diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,46 @@ --- +## [0.17.3] - 2026-08-01 + +Point release on the 0.17.x line, focused on **release-artifact integrity**. The 0.17.2 release shipped +two defects that every existing check missed because they validated only the generated `dist/` tree, never +the packaged artifact: the `hydra-build` sdist shipped 3 of 8 modules, and the published Java `hydra-kernel` +jar was a whole [#417](https://github.com/CategoricalData/hydra/issues/417) rename behind. This release adds +an artifact-level publish-completeness gate that inspects the actual uploaded archive, closing that class of +defect across all five registries. + +### Highlights + +- **Artifact-content completeness gate** ([#621](https://github.com/CategoricalData/hydra/issues/621)): + every publish path (Hackage sdist, Maven-Java jar, Maven-Scala jar, PyPI wheel, npm tarball) now inspects + the *packaged* archive and hard-fails if any module a package's manifest declares is missing. The gate also + asserts the post-#417 class names are present in the Java kernel jar. + +### Bug fixes + +- **PyPI wheels dropped non-`hydra.*` roots** ([#621](https://github.com/CategoricalData/hydra/issues/621)): + the wheel packaging hardcoded `packages = ["src/main/python/hydra"]`, silently omitting `hydra-pg`'s + `com.gdblab.*` and `openGql.*` modules. The generator now ships every emitted top-level root. + +### Improvements + +- **CI guard for dist-tree completeness** ([#524](https://github.com/CategoricalData/hydra/issues/524)): + asserts every manifest `mainModules` namespace is emitted into `dist/haskell`, catching the 0.17.2 + `hydra-build` truncation at the tree level (the artifact gate is the packaged-archive counterpart). +- **Manifest generation extracted** into `Hydra.ManifestGeneration`, and the structurally-unneeded #607 + sed shims dropped ([#622](https://github.com/CategoricalData/hydra/issues/622)). +- **Release-verification signing fix** ([#441](https://github.com/CategoricalData/hydra/issues/441)): + sign the reproducible uncompressed `.tar` and verify the `.asc` against the GitHub Release asset. +- **Java CI signing gate** ([#591](https://github.com/CategoricalData/hydra/issues/591)): `publishToMavenLocal` + skips gpg signing when no key is present. + +### Internal + +- Version bump to 0.17.3; `hostVersion` advanced to 0.17.2 and the temporary #417 Java/Python local-host + shims removed now that 0.17.2 is published on the registries + ([#417](https://github.com/CategoricalData/hydra/issues/417)). + ## [0.17.2] - 2026-07-28 Point release on the 0.17.x line. Themes: promotion of the generator's routing and manifest diff --git a/hydra-python.cabal b/hydra-python.cabal --- a/hydra-python.cabal +++ b/hydra-python.cabal @@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: hydra-python -version: 0.17.3 +version: 0.17.4 synopsis: Hydra's Python coder: emit Python source from Hydra modules description: Hydra is an implementation of the LambdaGraph data model, which takes advantage of an isomorphism between labeled hypergraphs and typed lambda calculus: in Hydra, "graphs are programs, and programs are graphs". Python support for Hydra category: Data @@ -47,6 +47,6 @@ build-depends: base >=4.19.0 && <4.22 , containers >=0.6.7 && <0.8 - , hydra-kernel ==0.17.3 + , hydra-kernel ==0.17.4 , scientific >=0.3.7 && <0.4 default-language: Haskell2010 diff --git a/src/main/haskell/Hydra/Dsl/Python/Environment.hs b/src/main/haskell/Hydra/Dsl/Python/Environment.hs --- a/src/main/haskell/Hydra/Dsl/Python/Environment.hs +++ b/src/main/haskell/Hydra/Dsl/Python/Environment.hs @@ -5,12 +5,7 @@ module Hydra.Dsl.Python.Environment where import qualified Hydra.Core as Core -import qualified Hydra.Dsl.Core as DslCore -import qualified Hydra.Dsl.Graph as DslGraph -import qualified Hydra.Dsl.Packaging as DslPackaging import qualified Hydra.Dsl.Python.Syntax as PythonSyntax -import qualified Hydra.Dsl.Typing as DslTyping -import qualified Hydra.Dsl.Util as DslUtil import qualified Hydra.Graph as Graph import qualified Hydra.Packaging as Packaging import qualified Hydra.Python.Environment as Environment @@ -93,8 +88,8 @@ Core.fieldTerm = (Typed.unTypedTerm newVal)}]})) -- | DSL constructor for hydra.python.environment.PythonEnvironment -pythonEnvironment :: Typed.TypedTerm (Util.ModuleNames Syntax.DottedName) -> Typed.TypedTerm ([Core.Name], (M.Map Core.Name Syntax.Name)) -> Typed.TypedTerm Graph.Graph -> Typed.TypedTerm (S.Set Core.Name) -> Typed.TypedTerm Environment.PythonVersion -> Typed.TypedTerm Bool -> Typed.TypedTerm (S.Set Core.Name) -> Typed.TypedTerm Environment.PythonEnvironment -pythonEnvironment namespaces boundTypeVariables graph nullaryBindings version skipCasts inlineVariables = +pythonEnvironment :: Typed.TypedTerm (Util.ModuleNames Syntax.DottedName) -> Typed.TypedTerm ([Core.Name], (M.Map Core.Name Syntax.Name)) -> Typed.TypedTerm Graph.Graph -> Typed.TypedTerm (S.Set Core.Name) -> Typed.TypedTerm Environment.PythonVersion -> Typed.TypedTerm Bool -> Typed.TypedTerm (S.Set Core.Name) -> Typed.TypedTerm (S.Set String) -> Typed.TypedTerm Environment.PythonEnvironment +pythonEnvironment namespaces boundTypeVariables graph nullaryBindings version skipCasts inlineVariables overlaySubs = Typed.TypedTerm (Core.TermRecord (Core.Record { Core.recordTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), Core.recordFields = [ @@ -118,7 +113,10 @@ Core.fieldTerm = (Typed.unTypedTerm skipCasts)}, Core.Field { Core.fieldName = (Core.Name "inlineVariables"), - Core.fieldTerm = (Typed.unTypedTerm inlineVariables)}]})) + Core.fieldTerm = (Typed.unTypedTerm inlineVariables)}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Typed.unTypedTerm overlaySubs)}]})) -- | DSL accessor for the boundTypeVariables field of hydra.python.environment.PythonEnvironment pythonEnvironmentBoundTypeVariables :: Typed.TypedTerm Environment.PythonEnvironment -> Typed.TypedTerm ([Core.Name], (M.Map Core.Name Syntax.Name)) @@ -165,6 +163,15 @@ Core.projectionFieldName = (Core.Name "nullaryBindings")})), Core.applicationArgument = (Typed.unTypedTerm x)})) +-- | DSL accessor for the overlaySubs field of hydra.python.environment.PythonEnvironment +pythonEnvironmentOverlaySubs :: Typed.TypedTerm Environment.PythonEnvironment -> Typed.TypedTerm (S.Set String) +pythonEnvironmentOverlaySubs x = + Typed.TypedTerm (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "overlaySubs")})), + Core.applicationArgument = (Typed.unTypedTerm x)})) + -- | DSL name token for hydra.python.environment.PythonEnvironment pythonEnvironmentPythonEnvironment :: Typed.TypedName Environment.PythonEnvironment pythonEnvironmentPythonEnvironment = Typed.TypedName (Core.Name "hydra.python.environment.PythonEnvironment") @@ -237,6 +244,13 @@ Core.applicationFunction = (Core.TermProject (Core.Projection { Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), Core.projectionFieldName = (Core.Name "inlineVariables")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "overlaySubs")})), Core.applicationArgument = (Typed.unTypedTerm original)}))}]})) -- | DSL updater for the graph field of hydra.python.environment.PythonEnvironment @@ -289,6 +303,13 @@ Core.applicationFunction = (Core.TermProject (Core.Projection { Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), Core.projectionFieldName = (Core.Name "inlineVariables")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "overlaySubs")})), Core.applicationArgument = (Typed.unTypedTerm original)}))}]})) -- | DSL updater for the inlineVariables field of hydra.python.environment.PythonEnvironment @@ -341,7 +362,14 @@ Core.applicationArgument = (Typed.unTypedTerm original)}))}, Core.Field { Core.fieldName = (Core.Name "inlineVariables"), - Core.fieldTerm = (Typed.unTypedTerm newVal)}]})) + Core.fieldTerm = (Typed.unTypedTerm newVal)}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "overlaySubs")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}]})) -- | DSL updater for the namespaces field of hydra.python.environment.PythonEnvironment pythonEnvironmentWithNamespaces :: Typed.TypedTerm Environment.PythonEnvironment -> Typed.TypedTerm (Util.ModuleNames Syntax.DottedName) -> Typed.TypedTerm Environment.PythonEnvironment @@ -393,6 +421,13 @@ Core.applicationFunction = (Core.TermProject (Core.Projection { Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), Core.projectionFieldName = (Core.Name "inlineVariables")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "overlaySubs")})), Core.applicationArgument = (Typed.unTypedTerm original)}))}]})) -- | DSL updater for the nullaryBindings field of hydra.python.environment.PythonEnvironment @@ -445,8 +480,74 @@ Core.applicationFunction = (Core.TermProject (Core.Projection { Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), Core.projectionFieldName = (Core.Name "inlineVariables")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "overlaySubs")})), Core.applicationArgument = (Typed.unTypedTerm original)}))}]})) +-- | DSL updater for the overlaySubs field of hydra.python.environment.PythonEnvironment +pythonEnvironmentWithOverlaySubs :: Typed.TypedTerm Environment.PythonEnvironment -> Typed.TypedTerm (S.Set String) -> Typed.TypedTerm Environment.PythonEnvironment +pythonEnvironmentWithOverlaySubs original newVal = + Typed.TypedTerm (Core.TermRecord (Core.Record { + Core.recordTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.recordFields = [ + Core.Field { + Core.fieldName = (Core.Name "namespaces"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "namespaces")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "boundTypeVariables"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "boundTypeVariables")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "graph"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "graph")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "nullaryBindings"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "nullaryBindings")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "version"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "version")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "skipCasts"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "skipCasts")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "inlineVariables"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "inlineVariables")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Typed.unTypedTerm newVal)}]})) + -- | DSL updater for the skipCasts field of hydra.python.environment.PythonEnvironment pythonEnvironmentWithSkipCasts :: Typed.TypedTerm Environment.PythonEnvironment -> Typed.TypedTerm Bool -> Typed.TypedTerm Environment.PythonEnvironment pythonEnvironmentWithSkipCasts original newVal = @@ -497,6 +598,13 @@ Core.applicationFunction = (Core.TermProject (Core.Projection { Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), Core.projectionFieldName = (Core.Name "inlineVariables")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "overlaySubs")})), Core.applicationArgument = (Typed.unTypedTerm original)}))}]})) -- | DSL updater for the version field of hydra.python.environment.PythonEnvironment @@ -549,6 +657,13 @@ Core.applicationFunction = (Core.TermProject (Core.Projection { Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), Core.projectionFieldName = (Core.Name "inlineVariables")})), + Core.applicationArgument = (Typed.unTypedTerm original)}))}, + Core.Field { + Core.fieldName = (Core.Name "overlaySubs"), + Core.fieldTerm = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermProject (Core.Projection { + Core.projectionTypeName = (Core.Name "hydra.python.environment.PythonEnvironment"), + Core.projectionFieldName = (Core.Name "overlaySubs")})), Core.applicationArgument = (Typed.unTypedTerm original)}))}]})) -- | DSL constructor for hydra.python.environment.PythonModuleMetadata diff --git a/src/main/haskell/Hydra/Dsl/Python/Names.hs b/src/main/haskell/Hydra/Dsl/Python/Names.hs --- a/src/main/haskell/Hydra/Dsl/Python/Names.hs +++ b/src/main/haskell/Hydra/Dsl/Python/Names.hs @@ -8,31 +8,8 @@ import qualified Hydra.Classes as Classes import qualified Hydra.Coders as Coders import qualified Hydra.Core as Core -import qualified Hydra.Dsl.Ast as DslAst -import qualified Hydra.Dsl.Coders as DslCoders -import qualified Hydra.Dsl.Core as DslCore -import qualified Hydra.Dsl.Error.Checking as ErrorChecking -import qualified Hydra.Dsl.Error.Core as DslErrorCore -import qualified Hydra.Dsl.Error.Packaging as DslErrorPackaging -import qualified Hydra.Dsl.Errors as DslErrors -import qualified Hydra.Dsl.Formatting as DslFormatting -import qualified Hydra.Dsl.Graph as DslGraph -import qualified Hydra.Dsl.Json.Model as JsonModel -import qualified Hydra.Dsl.Names as DslNames -import qualified Hydra.Dsl.Packaging as DslPackaging -import qualified Hydra.Dsl.Parsing as DslParsing -import qualified Hydra.Dsl.Paths as DslPaths import qualified Hydra.Dsl.Python.Environment as PythonEnvironment import qualified Hydra.Dsl.Python.Syntax as PythonSyntax -import qualified Hydra.Dsl.Query as DslQuery -import qualified Hydra.Dsl.Relational as DslRelational -import qualified Hydra.Dsl.Tabular as DslTabular -import qualified Hydra.Dsl.Testing as DslTesting -import qualified Hydra.Dsl.Topology as DslTopology -import qualified Hydra.Dsl.Typing as DslTyping -import qualified Hydra.Dsl.Util as DslUtil -import qualified Hydra.Dsl.Validation as DslValidation -import qualified Hydra.Dsl.Variants as DslVariants import qualified Hydra.Error.Checking as Checking import qualified Hydra.Error.Core as ErrorCore import qualified Hydra.Error.Packaging as ErrorPackaging @@ -62,6 +39,7 @@ import Prelude hiding (Enum, Ordering, decodeFloat, encodeFloat, fail, map, pure, sum) import qualified Data.Scientific as Sci import qualified Data.Map as M +import qualified Data.Set as S -- | DSL reference to hydra.python.names.encodeConstantForFieldName encodeConstantForFieldName :: Typed.TypedTerm t0 -> Typed.TypedTerm t1 -> Typed.TypedTerm Core.Name -> Typed.TypedTerm Syntax.Name @@ -131,18 +109,22 @@ Core.applicationArgument = (Typed.unTypedTerm arg0)})) -- | DSL reference to hydra.python.names.encodeNamespaceStringWithOverrides -encodeNamespaceStringWithOverrides :: Typed.TypedTerm Packaging.ModuleName -> Typed.TypedTerm String -encodeNamespaceStringWithOverrides arg0 = +encodeNamespaceStringWithOverrides :: Typed.TypedTerm (S.Set String) -> Typed.TypedTerm Packaging.ModuleName -> Typed.TypedTerm String +encodeNamespaceStringWithOverrides arg0 arg1 = Typed.TypedTerm (Core.TermApplication (Core.Application { - Core.applicationFunction = (Core.TermVariable (Core.Name "hydra.python.names.encodeNamespaceStringWithOverrides")), - Core.applicationArgument = (Typed.unTypedTerm arg0)})) + Core.applicationFunction = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermVariable (Core.Name "hydra.python.names.encodeNamespaceStringWithOverrides")), + Core.applicationArgument = (Typed.unTypedTerm arg0)})), + Core.applicationArgument = (Typed.unTypedTerm arg1)})) -- | DSL reference to hydra.python.names.encodeNamespaceWithOverrides -encodeNamespaceWithOverrides :: Typed.TypedTerm Packaging.ModuleName -> Typed.TypedTerm Syntax.DottedName -encodeNamespaceWithOverrides arg0 = +encodeNamespaceWithOverrides :: Typed.TypedTerm (S.Set String) -> Typed.TypedTerm Packaging.ModuleName -> Typed.TypedTerm Syntax.DottedName +encodeNamespaceWithOverrides arg0 arg1 = Typed.TypedTerm (Core.TermApplication (Core.Application { - Core.applicationFunction = (Core.TermVariable (Core.Name "hydra.python.names.encodeNamespaceWithOverrides")), - Core.applicationArgument = (Typed.unTypedTerm arg0)})) + Core.applicationFunction = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermVariable (Core.Name "hydra.python.names.encodeNamespaceWithOverrides")), + Core.applicationArgument = (Typed.unTypedTerm arg0)})), + Core.applicationArgument = (Typed.unTypedTerm arg1)})) -- | DSL reference to hydra.python.names.encodeTypeVariable encodeTypeVariable :: Typed.TypedTerm Core.Name -> Typed.TypedTerm Syntax.Name diff --git a/src/main/haskell/Hydra/Dsl/Python/Syntax.hs b/src/main/haskell/Hydra/Dsl/Python/Syntax.hs --- a/src/main/haskell/Hydra/Dsl/Python/Syntax.hs +++ b/src/main/haskell/Hydra/Dsl/Python/Syntax.hs @@ -5,7 +5,6 @@ module Hydra.Dsl.Python.Syntax where import qualified Hydra.Core as Core -import qualified Hydra.Dsl.Core as DslCore import qualified Hydra.Python.Syntax as Syntax import qualified Hydra.Typed as Typed import Prelude hiding (Enum, Ordering, decodeFloat, encodeFloat, fail, map, pure, sum) diff --git a/src/main/haskell/Hydra/Dsl/Python/Utils.hs b/src/main/haskell/Hydra/Dsl/Python/Utils.hs --- a/src/main/haskell/Hydra/Dsl/Python/Utils.hs +++ b/src/main/haskell/Hydra/Dsl/Python/Utils.hs @@ -9,32 +9,9 @@ import qualified Hydra.Classes as Classes import qualified Hydra.Coders as Coders import qualified Hydra.Core as Core -import qualified Hydra.Dsl.Analysis as DslAnalysis -import qualified Hydra.Dsl.Ast as DslAst -import qualified Hydra.Dsl.Coders as DslCoders -import qualified Hydra.Dsl.Core as DslCore -import qualified Hydra.Dsl.Error.Checking as ErrorChecking -import qualified Hydra.Dsl.Error.Core as DslErrorCore -import qualified Hydra.Dsl.Error.Packaging as DslErrorPackaging -import qualified Hydra.Dsl.Errors as DslErrors -import qualified Hydra.Dsl.Graph as DslGraph -import qualified Hydra.Dsl.Json.Model as JsonModel -import qualified Hydra.Dsl.Packaging as DslPackaging -import qualified Hydra.Dsl.Parsing as DslParsing -import qualified Hydra.Dsl.Paths as DslPaths import qualified Hydra.Dsl.Python.Environment as PythonEnvironment import qualified Hydra.Dsl.Python.Names as PythonNames import qualified Hydra.Dsl.Python.Syntax as PythonSyntax -import qualified Hydra.Dsl.Query as DslQuery -import qualified Hydra.Dsl.Relational as DslRelational -import qualified Hydra.Dsl.Serialization as DslSerialization -import qualified Hydra.Dsl.Tabular as DslTabular -import qualified Hydra.Dsl.Testing as DslTesting -import qualified Hydra.Dsl.Topology as DslTopology -import qualified Hydra.Dsl.Typing as DslTyping -import qualified Hydra.Dsl.Util as DslUtil -import qualified Hydra.Dsl.Validation as DslValidation -import qualified Hydra.Dsl.Variants as DslVariants import qualified Hydra.Error.Checking as Checking import qualified Hydra.Error.Core as ErrorCore import qualified Hydra.Error.Packaging as ErrorPackaging @@ -62,6 +39,7 @@ import qualified Hydra.Variants as Variants import Prelude hiding (Enum, Ordering, decodeFloat, encodeFloat, fail, map, pure, sum) import qualified Data.Scientific as Sci +import qualified Data.Set as S -- | DSL reference to hydra.python.utils.annotatedExpression annotatedExpression :: Typed.TypedTerm (Maybe String) -> Typed.TypedTerm Syntax.Expression -> Typed.TypedTerm Syntax.Expression @@ -169,13 +147,15 @@ Core.applicationArgument = (Typed.unTypedTerm arg0)})) -- | DSL reference to hydra.python.utils.findNamespaces -findNamespaces :: Typed.TypedTerm Packaging.ModuleName -> Typed.TypedTerm [Packaging.Definition] -> Typed.TypedTerm (Util.ModuleNames Syntax.DottedName) -findNamespaces arg0 arg1 = +findNamespaces :: Typed.TypedTerm (S.Set String) -> Typed.TypedTerm Packaging.ModuleName -> Typed.TypedTerm [Packaging.Definition] -> Typed.TypedTerm (Util.ModuleNames Syntax.DottedName) +findNamespaces arg0 arg1 arg2 = Typed.TypedTerm (Core.TermApplication (Core.Application { Core.applicationFunction = (Core.TermApplication (Core.Application { - Core.applicationFunction = (Core.TermVariable (Core.Name "hydra.python.utils.findNamespaces")), - Core.applicationArgument = (Typed.unTypedTerm arg0)})), - Core.applicationArgument = (Typed.unTypedTerm arg1)})) + Core.applicationFunction = (Core.TermApplication (Core.Application { + Core.applicationFunction = (Core.TermVariable (Core.Name "hydra.python.utils.findNamespaces")), + Core.applicationArgument = (Typed.unTypedTerm arg0)})), + Core.applicationArgument = (Typed.unTypedTerm arg1)})), + Core.applicationArgument = (Typed.unTypedTerm arg2)})) -- | DSL reference to hydra.python.utils.functionCall functionCall :: Typed.TypedTerm Syntax.Primary -> Typed.TypedTerm [Syntax.Expression] -> Typed.TypedTerm Syntax.Expression diff --git a/src/main/haskell/Hydra/Python/Coder.hs b/src/main/haskell/Hydra/Python/Coder.hs --- a/src/main/haskell/Hydra/Python/Coder.hs +++ b/src/main/haskell/Hydra/Python/Coder.hs @@ -850,14 +850,14 @@ Lists.map (\field -> (PythonNames.encodeConstantForFieldName env name (Core.fieldTypeName field), (Core.fieldTypeName field))) fields in (Lists.map toStmt (Lists.cons namePair fieldPairs)) --- | Encode a Hydra module to a Python module AST -encodePythonModule :: Typing.InferenceContext -> Graph.Graph -> Packaging.Module -> [Packaging.Definition] -> Either Errors.Error Syntax.Module -encodePythonModule cx g mod defs0 = +-- | Encode a Hydra module to a Python module AST, given the on-disk overlay-existence set (#630) driving emission-time primitive-reference redirects +encodePythonModule :: S.Set String -> Typing.InferenceContext -> Graph.Graph -> Packaging.Module -> [Packaging.Definition] -> Either Errors.Error Syntax.Module +encodePythonModule overlaySubs cx g mod defs0 = let defs = Environment.reorderDefs defs0 - meta0 = gatherMetadata (Packaging.moduleName mod) defs + meta0 = gatherMetadata overlaySubs (Packaging.moduleName mod) defs namespaces0 = PythonEnvironment.pythonModuleMetadataNamespaces meta0 - env0 = initialEnvironment namespaces0 g + env0 = initialEnvironment namespaces0 g overlaySubs isTypeMod = isTypeModuleCheck defs0 in (withDefinitions env0 defs (\env -> Eithers.bind (Eithers.map (\xs -> Lists.concat xs) (Eithers.mapList (\d -> encodeDefinition cx env d) defs)) (\defStmts -> let meta2 = Logic.ifElse (Logic.and (Logic.not isTypeMod) useInlineTypeParams) (setMetaUsesTypeVar meta0 False) meta0 @@ -974,7 +974,8 @@ PythonEnvironment.pythonEnvironmentNullaryBindings = (PythonEnvironment.pythonEnvironmentNullaryBindings innerEnv0), PythonEnvironment.pythonEnvironmentVersion = (PythonEnvironment.pythonEnvironmentVersion innerEnv0), PythonEnvironment.pythonEnvironmentSkipCasts = (PythonEnvironment.pythonEnvironmentSkipCasts innerEnv0), - PythonEnvironment.pythonEnvironmentInlineVariables = (Sets.union (Sets.fromList bindingNames) (PythonEnvironment.pythonEnvironmentInlineVariables innerEnv0))} + PythonEnvironment.pythonEnvironmentInlineVariables = (Sets.union (Sets.fromList bindingNames) (PythonEnvironment.pythonEnvironmentInlineVariables innerEnv0)), + PythonEnvironment.pythonEnvironmentOverlaySubs = (PythonEnvironment.pythonEnvironmentOverlaySubs innerEnv0)} in (Eithers.bind (encodeTermInline cx innerEnv False innerBody) (\pbody -> let pparams = Lists.map (PythonNames.encodeName False Util.CaseConventionLowerSnake innerEnv) params in (Logic.ifElse (Lists.null bindings) (Right (makeUncurriedLambda pparams pbody)) (Eithers.bind (Eithers.mapList (encodeBindingAsAssignment cx False innerEnv) bindings) (\pbindingExprs -> @@ -1512,7 +1513,8 @@ PythonEnvironment.pythonEnvironmentNullaryBindings = (PythonEnvironment.pythonEnvironmentNullaryBindings env), PythonEnvironment.pythonEnvironmentVersion = (PythonEnvironment.pythonEnvironmentVersion env), PythonEnvironment.pythonEnvironmentSkipCasts = (PythonEnvironment.pythonEnvironmentSkipCasts env), - PythonEnvironment.pythonEnvironmentInlineVariables = (PythonEnvironment.pythonEnvironmentInlineVariables env)} + PythonEnvironment.pythonEnvironmentInlineVariables = (PythonEnvironment.pythonEnvironmentInlineVariables env), + PythonEnvironment.pythonEnvironmentOverlaySubs = (PythonEnvironment.pythonEnvironmentOverlaySubs env)} -- | Extend metadata based on a term (used during module encoding) extendMetaForTerm :: Bool -> PythonEnvironment.PythonModuleMetadata -> Core.Term -> PythonEnvironment.PythonModuleMetadata @@ -1581,7 +1583,7 @@ let names = Sets.unions (Lists.map (\t -> Dependencies.typeDependencyNames False t) types) currentNs = PythonEnvironment.pythonModuleMetadataNamespaces meta - updatedNs = Analysis.addNamesToModuleNames PythonNames.encodeNamespaceWithOverrides names currentNs + updatedNs = Analysis.addNamesToModuleNames (PythonNames.encodeNamespaceWithOverrides Sets.empty) names currentNs meta1 = setMetaNamespaces updatedNs meta in (Lists.foldl (\m -> \t -> extendMetaForType True False t m) meta1 types) @@ -1664,10 +1666,10 @@ in (go [] term) -- | Gather metadata from definitions -gatherMetadata :: Packaging.ModuleName -> [Packaging.Definition] -> PythonEnvironment.PythonModuleMetadata -gatherMetadata focusNs defs = +gatherMetadata :: S.Set String -> Packaging.ModuleName -> [Packaging.Definition] -> PythonEnvironment.PythonModuleMetadata +gatherMetadata overlaySubs focusNs defs = - let start = emptyMetadata (Utils.findNamespaces focusNs defs) + let start = emptyMetadata (Utils.findNamespaces overlaySubs focusNs defs) addDef = \meta -> \def -> case def of Packaging.DefinitionTerm v0 -> @@ -1710,9 +1712,9 @@ Syntax.powerRhs = Nothing}))}}}}}}, Syntax.comparisonRhs = []})]])) tparamList)))) --- | Create an initial Python environment for code generation -initialEnvironment :: Util.ModuleNames Syntax.DottedName -> Graph.Graph -> PythonEnvironment.PythonEnvironment -initialEnvironment namespaces tcontext = +-- | Create an initial Python environment for code generation, given the on-disk overlay-existence set (#630) driving emission-time primitive-reference redirects +initialEnvironment :: Util.ModuleNames Syntax.DottedName -> Graph.Graph -> S.Set String -> PythonEnvironment.PythonEnvironment +initialEnvironment namespaces tcontext overlaySubs = PythonEnvironment.PythonEnvironment { PythonEnvironment.pythonEnvironmentNamespaces = namespaces, PythonEnvironment.pythonEnvironmentBoundTypeVariables = ([], Maps.empty), @@ -1720,7 +1722,8 @@ PythonEnvironment.pythonEnvironmentNullaryBindings = Sets.empty, PythonEnvironment.pythonEnvironmentVersion = targetPythonVersion, PythonEnvironment.pythonEnvironmentSkipCasts = True, - PythonEnvironment.pythonEnvironmentInlineVariables = Sets.empty} + PythonEnvironment.pythonEnvironmentInlineVariables = Sets.empty, + PythonEnvironment.pythonEnvironmentOverlaySubs = overlaySubs} -- | Create initial empty metadata for a Python module initialMetadata :: Packaging.ModuleName -> PythonEnvironment.PythonModuleMetadata @@ -1965,9 +1968,9 @@ in (Lists.map (\p -> standardImportStatement (Pairs.first p) (Pairs.second p)) simplified) -- | Convert a Hydra module to Python source files -moduleToPython :: Packaging.Module -> [Packaging.Definition] -> Typing.InferenceContext -> Graph.Graph -> Either Errors.Error (M.Map String String) -moduleToPython mod defs cx g = - Eithers.bind (encodePythonModule cx g mod defs) (\file -> +moduleToPython :: S.Set String -> Packaging.Module -> [Packaging.Definition] -> Typing.InferenceContext -> Graph.Graph -> Either Errors.Error (M.Map String String) +moduleToPython overlaySubs mod defs cx g = + Eithers.bind (encodePythonModule overlaySubs cx g mod defs) (\file -> let s = Serialization.printExpr (Serialization.parenthesize (Serde.moduleToExpr file)) path = Names.moduleNameToFilePath Util.CaseConventionLowerSnake (File.FileExtension "py") (Packaging.moduleName mod) in (Right (Maps.singleton path s))) @@ -2003,7 +2006,8 @@ PythonEnvironment.pythonEnvironmentNullaryBindings = (PythonEnvironment.pythonEnvironmentNullaryBindings env), PythonEnvironment.pythonEnvironmentVersion = (PythonEnvironment.pythonEnvironmentVersion env), PythonEnvironment.pythonEnvironmentSkipCasts = (PythonEnvironment.pythonEnvironmentSkipCasts env), - PythonEnvironment.pythonEnvironmentInlineVariables = (PythonEnvironment.pythonEnvironmentInlineVariables env)} + PythonEnvironment.pythonEnvironmentInlineVariables = (PythonEnvironment.pythonEnvironmentInlineVariables env), + PythonEnvironment.pythonEnvironmentOverlaySubs = (PythonEnvironment.pythonEnvironmentOverlaySubs env)} -- | Build a fluent builder (factory + nested Builder dataclass) for a record recordBuilderClass :: PythonEnvironment.PythonEnvironment -> Core.Name -> [Core.FieldType] -> Maybe Syntax.Args -> Either t0 [Syntax.Statement] @@ -2940,7 +2944,8 @@ PythonEnvironment.pythonEnvironmentNullaryBindings = (PythonEnvironment.pythonEnvironmentNullaryBindings innerEnv), PythonEnvironment.pythonEnvironmentVersion = (PythonEnvironment.pythonEnvironmentVersion innerEnv), PythonEnvironment.pythonEnvironmentSkipCasts = (PythonEnvironment.pythonEnvironmentSkipCasts innerEnv), - PythonEnvironment.pythonEnvironmentInlineVariables = (Sets.union inlineVars (PythonEnvironment.pythonEnvironmentInlineVariables innerEnv))} + PythonEnvironment.pythonEnvironmentInlineVariables = (Sets.union inlineVars (PythonEnvironment.pythonEnvironmentInlineVariables innerEnv)), + PythonEnvironment.pythonEnvironmentOverlaySubs = (PythonEnvironment.pythonEnvironmentOverlaySubs innerEnv)} in (body updatedEnv))) -- | Execute a computation with type lambda context diff --git a/src/main/haskell/Hydra/Python/Environment.hs b/src/main/haskell/Hydra/Python/Environment.hs --- a/src/main/haskell/Hydra/Python/Environment.hs +++ b/src/main/haskell/Hydra/Python/Environment.hs @@ -43,7 +43,9 @@ -- | When True, skip generating cast() calls for reduced memory usage pythonEnvironmentSkipCasts :: Bool, -- | Variables that are inline let bindings (walrus operators) - pythonEnvironmentInlineVariables :: (S.Set Core.Name)} + pythonEnvironmentInlineVariables :: (S.Set Core.Name), + -- | hydra.lib. sub-namespaces that have an overlay implementation on this host (#630) -- the on-disk existence signal driving emission-time redirect of primitive references to hydra.overlay.python.lib. + pythonEnvironmentOverlaySubs :: (S.Set String)} _PythonEnvironment = Core.Name "hydra.python.environment.PythonEnvironment" @@ -60,6 +62,8 @@ _PythonEnvironment_skipCasts = Core.Name "skipCasts" _PythonEnvironment_inlineVariables = Core.Name "inlineVariables" + +_PythonEnvironment_overlaySubs = Core.Name "overlaySubs" -- | Temporary metadata used to create the header section of a Python file data PythonModuleMetadata = diff --git a/src/main/haskell/Hydra/Python/Names.hs b/src/main/haskell/Hydra/Python/Names.hs --- a/src/main/haskell/Hydra/Python/Names.hs +++ b/src/main/haskell/Hydra/Python/Names.hs @@ -21,6 +21,7 @@ import qualified Hydra.Overlay.Haskell.Lib.Maps as Maps import qualified Hydra.Overlay.Haskell.Lib.Optionals as Optionals import qualified Hydra.Overlay.Haskell.Lib.Pairs as Pairs +import qualified Hydra.Overlay.Haskell.Lib.Sets as Sets import qualified Hydra.Overlay.Haskell.Lib.Strings as Strings import qualified Hydra.Names as Names import qualified Hydra.Packaging as Packaging @@ -43,6 +44,7 @@ import Prelude hiding (Enum, Ordering, decodeFloat, encodeFloat, fail, map, pure, sum) import qualified Data.Scientific as Sci import qualified Data.Map as M +import qualified Data.Set as S -- | Generate a constant name for a field definition encodeConstantForFieldName :: t0 -> t1 -> Core.Name -> Syntax.Name @@ -73,7 +75,7 @@ mns = Util.qualifiedNameModuleName qualName local = Util.qualifiedNameLocal qualName pyLocal = sanitizePythonName (Formatting.convertCase Util.CaseConventionCamel conv local) - pyNs = \nsVal -> encodeNamespaceStringWithOverrides nsVal + pyNs = \nsVal -> encodeNamespaceStringWithOverrides (Environment.pythonEnvironmentOverlaySubs env) nsVal in (Logic.ifElse isQualified (Optionals.cases (Maps.lookup name boundVars) (Logic.ifElse (Equality.equal mns (Just focusNs)) (Syntax.Name (Logic.ifElse useFutureAnnotations pyLocal (Serde.escapePythonString True pyLocal))) (Optionals.cases mns (Syntax.Name pyLocal) (\nsVal -> Syntax.Name (Strings.concat2 (pyNs nsVal) (Strings.concat2 "." pyLocal))))) (\n -> n)) (Syntax.Name pyLocal)) -- | Encode a name as a fully qualified Python name @@ -87,7 +89,7 @@ qualName = Names.qualifyName name mns = Util.qualifiedNameModuleName qualName local = Util.qualifiedNameLocal qualName - pyNs = \nsVal -> encodeNamespaceStringWithOverrides nsVal + pyNs = \nsVal -> encodeNamespaceStringWithOverrides (Environment.pythonEnvironmentOverlaySubs env) nsVal in (Optionals.cases (Maps.lookup name boundVars) (Logic.ifElse (Equality.equal mns (Just focusNs)) (Syntax.Name (Logic.ifElse useFutureAnnotations local (Serde.escapePythonString True local))) (Optionals.cases mns (Syntax.Name (sanitizePythonName local)) (\nsVal -> Syntax.Name (Strings.concat2 (pyNs nsVal) (Strings.concat2 "." (sanitizePythonName local)))))) (\n -> n)) -- | Encode a namespace as a Python dotted name @@ -96,14 +98,16 @@ Syntax.DottedName (Lists.map (\part -> Syntax.Name (Formatting.convertCase Util.CaseConventionCamel Util.CaseConventionLowerSnake part)) (Strings.splitOn "." (Packaging.unModuleName nsVal))) -- | Convert a ModuleName to its Python dotted import string, routing overlay modules to hydra.overlay.python.* -encodeNamespaceStringWithOverrides :: Packaging.ModuleName -> String -encodeNamespaceStringWithOverrides nsVal = - Optionals.withDefault (Strings.join "." (Lists.map (Formatting.convertCase Util.CaseConventionCamel Util.CaseConventionLowerSnake) (Strings.splitOn "." (Packaging.unModuleName nsVal)))) (Maps.lookup nsVal overlayPythonModuleAliases) +encodeNamespaceStringWithOverrides :: S.Set String -> Packaging.ModuleName -> String +encodeNamespaceStringWithOverrides overlaySubs nsVal = + Optionals.cases (Logic.ifElse (Logic.and (Logic.and (Equality.equal (Lists.length (Strings.splitOn "." (Packaging.unModuleName nsVal))) 3) (Equality.equal (Lists.take 2 (Strings.splitOn "." (Packaging.unModuleName nsVal))) [ + "hydra", + "lib"])) (Sets.member (Strings.join "." (Lists.drop 2 (Strings.splitOn "." (Packaging.unModuleName nsVal)))) overlaySubs)) (Just (Strings.concat2 "hydra.overlay.python.lib." (Strings.join "." (Lists.drop 2 (Strings.splitOn "." (Packaging.unModuleName nsVal)))))) Nothing) (Optionals.withDefault (Strings.join "." (Lists.map (Formatting.convertCase Util.CaseConventionCamel Util.CaseConventionLowerSnake) (Strings.splitOn "." (Packaging.unModuleName nsVal)))) (Maps.lookup nsVal overlayPythonModuleAliases)) (\redirected -> redirected) -- | Encode a namespace as a Python dotted name, routing overlay modules to their hydra.overlay.python.* paths -encodeNamespaceWithOverrides :: Packaging.ModuleName -> Syntax.DottedName -encodeNamespaceWithOverrides nsVal = - Syntax.DottedName (Lists.map (\part -> Syntax.Name part) (Strings.splitOn "." (encodeNamespaceStringWithOverrides nsVal))) +encodeNamespaceWithOverrides :: S.Set String -> Packaging.ModuleName -> Syntax.DottedName +encodeNamespaceWithOverrides overlaySubs nsVal = + Syntax.DottedName (Lists.map (\part -> Syntax.Name part) (Strings.splitOn "." (encodeNamespaceStringWithOverrides overlaySubs nsVal))) -- | Encode a type variable name (capitalized) encodeTypeVariable :: Core.Name -> Syntax.Name diff --git a/src/main/haskell/Hydra/Python/Utils.hs b/src/main/haskell/Hydra/Python/Utils.hs --- a/src/main/haskell/Hydra/Python/Utils.hs +++ b/src/main/haskell/Hydra/Python/Utils.hs @@ -42,6 +42,7 @@ import qualified Hydra.Variants as Variants import Prelude hiding (Enum, Ordering, decodeFloat, encodeFloat, fail, map, pure, sum) import qualified Data.Scientific as Sci +import qualified Data.Set as S -- | Annotate an expression with an optional comment using Annotated[] annotatedExpression :: Maybe String -> Syntax.Expression -> Syntax.Expression @@ -153,12 +154,12 @@ doubleQuotedString :: String -> Syntax.Expression doubleQuotedString s = stringToPyExpression Syntax.QuoteStyleDouble s --- | Find all namespaces referenced by a list of definitions, plus the core namespace -findNamespaces :: Packaging.ModuleName -> [Packaging.Definition] -> Util.ModuleNames Syntax.DottedName -findNamespaces focusNs defs = +-- | Find all namespaces referenced by a list of definitions, plus the core namespace (#630: overlaySubs drives emission-time hydra.lib. redirect for these namespace refs) +findNamespaces :: S.Set String -> Packaging.ModuleName -> [Packaging.Definition] -> Util.ModuleNames Syntax.DottedName +findNamespaces overlaySubs focusNs defs = let coreNs = Packaging.ModuleName "hydra.core" - namespaces = Analysis.moduleNamesForDefinitions Names.encodeNamespaceWithOverrides focusNs defs + namespaces = Analysis.moduleNamesForDefinitions (Names.encodeNamespaceWithOverrides overlaySubs) focusNs defs in (Logic.ifElse (Equality.equal (Packaging.unModuleName (Pairs.first (Util.moduleNamesFocus namespaces))) (Packaging.unModuleName coreNs)) namespaces (Util.ModuleNames { Util.moduleNamesFocus = (Util.moduleNamesFocus namespaces), Util.moduleNamesMapping = (Maps.insert coreNs (Names.encodeNamespace coreNs) (Util.moduleNamesMapping namespaces))}))