futhark 0.25.2 → 0.25.3

raw patch · 251 files changed

+10911/−9253 lines, 251 filesdep ~bytestringPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependency ranges changed: bytestring

API changes (from Hackage documentation)

Files

• docs/c-api.rst +15/−14
• docs/index.rst +4/−3
• docs/man/futhark-hip.rst +128/−0
• docs/usage.rst +1/−1
• futhark.cabal +17/−8
• prelude/array.fut +2/−2
• rts/c/atomics.h +184/−148
• rts/c/backends/cuda.h +241/−144
• rts/c/backends/hip.h +911/−0
• rts/c/backends/opencl.h +301/−199
• rts/c/context_prototypes.h +21/−0
• rts/c/copy.h +262/−0
• rts/c/gpu.h +591/−0
• rts/c/gpu_prototypes.h +12/−0
• rts/c/half.h +3/−3
• rts/c/ispc_util.h +0/−54
• rts/c/scalar.h +2974/−2974
• rts/c/scalar_f16.h +194/−194
• rts/c/util.h +90/−0
• rts/cuda/prelude.cu +102/−0
• rts/opencl/copy.cl +83/−0
• rts/opencl/prelude.cl +54/−0
• rts/opencl/transpose.cl +273/−0
• rts/python/memory.py +132/−0
• rts/python/opencl.py +173/−1
• src/Futhark/AD/Fwd.hs +11/−11
• src/Futhark/AD/Rev.hs +2/−2
• src/Futhark/AD/Rev/Hist.hs +2/−2
• src/Futhark/AD/Rev/Loop.hs +17/−17
• src/Futhark/AD/Rev/Map.hs +1/−1
• src/Futhark/AD/Rev/Monad.hs +2/−2
• src/Futhark/AD/Rev/Reduce.hs +1/−1
• src/Futhark/Actions.hs +36/−3
• src/Futhark/Analysis/Alias.hs +7/−7
• src/Futhark/Analysis/CallGraph.hs +2/−2
• src/Futhark/Analysis/DataDependencies.hs +2/−2
• src/Futhark/Analysis/HORep/MapNest.hs +1/−1
• src/Futhark/Analysis/HORep/SOAC.hs +5/−5
• src/Futhark/Analysis/Interference.hs +26/−26
• src/Futhark/Analysis/LastUse.hs +15/−15
• src/Futhark/Analysis/MemAlias.hs +1/−1
• src/Futhark/Analysis/Metrics.hs +11/−11
• src/Futhark/Analysis/PrimExp.hs +21/−21
• src/Futhark/Analysis/PrimExp/Convert.hs +6/−6
• src/Futhark/Analysis/PrimExp/Simplify.hs +3/−3
• src/Futhark/Analysis/SymbolTable.hs +14/−14
• src/Futhark/Analysis/UsageTable.hs +5/−5
• src/Futhark/Bench.hs +1/−1
• src/Futhark/Builder.hs +9/−9
• src/Futhark/Builder/Class.hs +11/−11
• src/Futhark/CLI/Dataset.hs +1/−1
• src/Futhark/CLI/Dev.hs +1/−1
• src/Futhark/CLI/Eval.hs +1/−1
• src/Futhark/CLI/HIP.hs +17/−0
• src/Futhark/CLI/Main.hs +2/−0
• src/Futhark/CLI/REPL.hs +1/−1
• src/Futhark/CLI/Run.hs +1/−1
• src/Futhark/CLI/Test.hs +3/−3
• src/Futhark/CodeGen/Backends/CCUDA.hs +74/−335
• src/Futhark/CodeGen/Backends/CCUDA/Boilerplate.hs +0/−132
• src/Futhark/CodeGen/Backends/COpenCL.hs +131/−329
• src/Futhark/CodeGen/Backends/COpenCL/Boilerplate.hs +0/−320
• src/Futhark/CodeGen/Backends/GPU.hs +440/−0
• src/Futhark/CodeGen/Backends/GenericC.hs +50/−14
• src/Futhark/CodeGen/Backends/GenericC/Code.hs +126/−57
• src/Futhark/CodeGen/Backends/GenericC/Fun.hs +8/−4
• src/Futhark/CodeGen/Backends/GenericC/Monad.hs +23/−4
• src/Futhark/CodeGen/Backends/GenericC/Types.hs +12/−6
• src/Futhark/CodeGen/Backends/GenericPython.hs +149/−57
• src/Futhark/CodeGen/Backends/GenericPython/AST.hs +19/−9
• src/Futhark/CodeGen/Backends/HIP.hs +130/−0
• src/Futhark/CodeGen/Backends/MulticoreC.hs +7/−7
• src/Futhark/CodeGen/Backends/MulticoreISPC.hs +27/−40
• src/Futhark/CodeGen/Backends/MulticoreWASM.hs +1/−1
• src/Futhark/CodeGen/Backends/PyOpenCL.hs +103/−87
• src/Futhark/CodeGen/Backends/SequentialC.hs +1/−1
• src/Futhark/CodeGen/Backends/SequentialPython.hs +1/−1
• src/Futhark/CodeGen/Backends/SequentialWASM.hs +1/−1
• src/Futhark/CodeGen/ImpCode.hs +52/−34
• src/Futhark/CodeGen/ImpCode/OpenCL.hs +2/−3
• src/Futhark/CodeGen/ImpGen.hs +68/−211
• src/Futhark/CodeGen/ImpGen/CUDA.hs +1/−1
• src/Futhark/CodeGen/ImpGen/GPU.hs +8/−335
• src/Futhark/CodeGen/ImpGen/GPU/Base.hs +42/−29
• src/Futhark/CodeGen/ImpGen/GPU/Copy.hs +0/−81
• src/Futhark/CodeGen/ImpGen/GPU/Group.hs +57/−44
• src/Futhark/CodeGen/ImpGen/GPU/SegHist.hs +2/−2
• src/Futhark/CodeGen/ImpGen/GPU/SegRed.hs +6/−8
• src/Futhark/CodeGen/ImpGen/GPU/SegScan.hs +34/−4
• src/Futhark/CodeGen/ImpGen/GPU/SegScan/SinglePass.hs +30/−23
• src/Futhark/CodeGen/ImpGen/GPU/SegScan/TwoPass.hs +2/−4
• src/Futhark/CodeGen/ImpGen/GPU/ToOpenCL.hs +82/−207
• src/Futhark/CodeGen/ImpGen/GPU/Transpose.hs +0/−425
• src/Futhark/CodeGen/ImpGen/HIP.hs +17/−0
• src/Futhark/CodeGen/ImpGen/Multicore.hs +1/−1
• src/Futhark/CodeGen/ImpGen/Multicore/Base.hs +1/−1
• src/Futhark/CodeGen/ImpGen/Multicore/SegRed.hs +1/−1
• src/Futhark/CodeGen/ImpGen/OpenCL.hs +1/−1
• src/Futhark/CodeGen/ImpGen/Sequential.hs +1/−1
• src/Futhark/CodeGen/ImpGen/Transpose.hs +0/−219
• src/Futhark/CodeGen/RTS/C.hs +24/−0
• src/Futhark/CodeGen/RTS/CUDA.hs +12/−0
• src/Futhark/CodeGen/RTS/OpenCL.hs +27/−0
• src/Futhark/Compiler.hs +4/−4
• src/Futhark/Compiler/Program.hs +3/−3
• src/Futhark/Construct.hs +32/−32
• src/Futhark/Error.hs +3/−3
• src/Futhark/IR/Aliases.hs +14/−14
• src/Futhark/IR/GPU/Op.hs +9/−9
• src/Futhark/IR/GPU/Simplify.hs +2/−2
• src/Futhark/IR/GPUMem.hs +2/−2
• src/Futhark/IR/MC/Op.hs +6/−6
• src/Futhark/IR/MCMem.hs +1/−1
• src/Futhark/IR/Mem.hs +36/−32
• src/Futhark/IR/Mem/Interval.hs +1/−1
• src/Futhark/IR/Mem/IxFun.hs +35/−136
• src/Futhark/IR/Mem/LMAD.hs +83/−219
• src/Futhark/IR/Mem/Simplify.hs +15/−10
• src/Futhark/IR/Parse.hs +4/−5
• src/Futhark/IR/Pretty.hs +33/−27
• src/Futhark/IR/Prop.hs +5/−5
• src/Futhark/IR/Prop/Aliases.hs +8/−8
• src/Futhark/IR/Prop/Constants.hs +1/−1
• src/Futhark/IR/Prop/Names.hs +23/−23
• src/Futhark/IR/Prop/Patterns.hs +7/−7
• src/Futhark/IR/Prop/Rearrange.hs +2/−2
• src/Futhark/IR/Prop/Reshape.hs +5/−5
• src/Futhark/IR/Prop/Scope.hs +9/−9
• src/Futhark/IR/Prop/TypeOf.hs +6/−6
• src/Futhark/IR/Prop/Types.hs +25/−25
• src/Futhark/IR/Rephrase.hs +11/−11
• src/Futhark/IR/RetType.hs +2/−2
• src/Futhark/IR/SOACS.hs +1/−1
• src/Futhark/IR/SOACS/SOAC.hs +88/−58
• src/Futhark/IR/SOACS/Simplify.hs +5/−5
• src/Futhark/IR/SegOp.hs +61/−54
• src/Futhark/IR/SeqMem.hs +1/−1
• src/Futhark/IR/Syntax.hs +20/−20
• src/Futhark/IR/Syntax/Core.hs +3/−3
• src/Futhark/IR/Traversals.hs +16/−16
• src/Futhark/IR/TypeCheck.hs +65/−66
• src/Futhark/Internalise/Bindings.hs +3/−3
• src/Futhark/Internalise/Defunctionalise.hs +10/−10
• src/Futhark/Internalise/Defunctorise.hs +6/−9
• src/Futhark/Internalise/Exps.hs +5/−5
• src/Futhark/Internalise/FullNormalise.hs +6/−6
• src/Futhark/Internalise/LiftLambdas.hs +12/−3
• src/Futhark/Internalise/Monad.hs +1/−1
• src/Futhark/Internalise/Monomorphise.hs +7/−7
• src/Futhark/Internalise/ReplaceRecords.hs +1/−1
• src/Futhark/Internalise/TypesValues.hs +1/−1
• src/Futhark/MonadFreshNames.hs +14/−14
• src/Futhark/Optimise/ArrayShortCircuiting.hs +2/−2
• src/Futhark/Optimise/ArrayShortCircuiting/ArrayCoalescing.hs +39/−41
• src/Futhark/Optimise/ArrayShortCircuiting/MemRefAggreg.hs +10/−11
• src/Futhark/Optimise/ArrayShortCircuiting/TopdownAnalysis.hs +7/−8
• src/Futhark/Optimise/BlkRegTiling.hs +3/−5
• src/Futhark/Optimise/CSE.hs +4/−4
• src/Futhark/Optimise/DoubleBuffer.hs +18/−18
• src/Futhark/Optimise/EntryPointMem.hs +4/−4
• src/Futhark/Optimise/Fusion.hs +3/−3
• src/Futhark/Optimise/Fusion/Composing.hs +4/−4
• src/Futhark/Optimise/Fusion/GraphRep.hs +16/−16
• src/Futhark/Optimise/Fusion/TryFusion.hs +1/−1
• src/Futhark/Optimise/GenRedOpt.hs +1/−1
• src/Futhark/Optimise/HistAccs.hs +2/−2
• src/Futhark/Optimise/InPlaceLowering.hs +10/−10
• src/Futhark/Optimise/InPlaceLowering/LowerIntoStm.hs +5/−5
• src/Futhark/Optimise/InPlaceLowering/SubstituteIndices.hs +3/−3
• src/Futhark/Optimise/InliningDeadFun.hs +8/−8
• src/Futhark/Optimise/MemoryBlockMerging.hs +8/−8
• src/Futhark/Optimise/MemoryBlockMerging/GreedyColoring.hs +2/−2
• src/Futhark/Optimise/MergeGPUBodies.hs +5/−5
• src/Futhark/Optimise/ReduceDeviceSyncs.hs +3/−3
• src/Futhark/Optimise/ReduceDeviceSyncs/MigrationTable.hs +14/−14
• src/Futhark/Optimise/ReduceDeviceSyncs/MigrationTable/Graph.hs +2/−2
• src/Futhark/Optimise/Simplify.hs +1/−1
• src/Futhark/Optimise/Simplify/Engine.hs +58/−64
• src/Futhark/Optimise/Simplify/Rep.hs +20/−20
• src/Futhark/Optimise/Simplify/Rule.hs +14/−14
• src/Futhark/Optimise/Simplify/Rules.hs +7/−7
• src/Futhark/Optimise/Simplify/Rules/BasicOp.hs +18/−11
• src/Futhark/Optimise/Simplify/Rules/ClosedForm.hs +1/−1
• src/Futhark/Optimise/Simplify/Rules/Index.hs +1/−1
• src/Futhark/Optimise/Simplify/Rules/Loop.hs +15/−17
• src/Futhark/Optimise/Simplify/Rules/Match.hs +4/−4
• src/Futhark/Optimise/Sink.hs +12/−12
• src/Futhark/Optimise/TileLoops.hs +9/−9
• src/Futhark/Optimise/TileLoops/Shared.hs +3/−3
• src/Futhark/Optimise/Unstream.hs +7/−7
• src/Futhark/Pass.hs +1/−1
• src/Futhark/Pass/ExpandAllocations.hs +64/−66
• src/Futhark/Pass/ExplicitAllocations.hs +26/−51
• src/Futhark/Pass/ExplicitAllocations/GPU.hs +19/−13
• src/Futhark/Pass/ExplicitAllocations/SegOp.hs +7/−8
• src/Futhark/Pass/ExtractKernels.hs +8/−8
• src/Futhark/Pass/ExtractKernels/BlockedKernel.hs +2/−2
• src/Futhark/Pass/ExtractKernels/DistributeNests.hs +13/−13
• src/Futhark/Pass/ExtractKernels/Distribution.hs +7/−7
• src/Futhark/Pass/ExtractKernels/ISRWIM.hs +1/−1
• src/Futhark/Pass/ExtractKernels/Interchange.hs +1/−1
• src/Futhark/Pass/ExtractKernels/Intragroup.hs +22/−8
• src/Futhark/Pass/ExtractKernels/StreamKernel.hs +1/−1
• src/Futhark/Pass/ExtractMulticore.hs +4/−4
• src/Futhark/Pass/FirstOrderTransform.hs +1/−1
• src/Futhark/Pass/KernelBabysitting.hs +4/−4
• src/Futhark/Pass/LiftAllocations.hs +2/−2
• src/Futhark/Pass/LowerAllocations.hs +2/−2
• src/Futhark/Pipeline.hs +3/−3
• src/Futhark/Pkg/Info.hs +5/−5
• src/Futhark/Pkg/Solve.hs +1/−1
• src/Futhark/Script.hs +2/−2
• src/Futhark/Test.hs +8/−8
• src/Futhark/Test/Values.hs +1/−1
• src/Futhark/Transform/CopyPropagate.hs +1/−1
• src/Futhark/Transform/FirstOrderTransform.hs +9/−9
• src/Futhark/Transform/Rename.hs +19/−19
• src/Futhark/Transform/Substitute.hs +22/−22
• src/Futhark/Util.hs +5/−5
• src/Futhark/Util/IntegralExp.hs +4/−4
• src/Futhark/Util/Log.hs +4/−4
• src/Futhark/Util/Options.hs +1/−1
• src/Futhark/Util/Pretty.hs +6/−6
• src/Language/Futhark/Core.hs +2/−2
• src/Language/Futhark/FreeVars.hs +1/−1
• src/Language/Futhark/Interpreter.hs +76/−28
• src/Language/Futhark/Interpreter/Values.hs +3/−3
• src/Language/Futhark/Parser/Lexer/Tokens.hs +8/−8
• src/Language/Futhark/Parser/Monad.hs +2/−2
• src/Language/Futhark/Parser/Parser.y +2/−2
• src/Language/Futhark/Pretty.hs +42/−27
• src/Language/Futhark/Primitive.hs +6/−6
• src/Language/Futhark/Prop.hs +3/−3
• src/Language/Futhark/Query.hs +3/−3
• src/Language/Futhark/Syntax.hs +22/−22
• src/Language/Futhark/Traversals.hs +33/−46
• src/Language/Futhark/TypeChecker.hs +11/−7
• src/Language/Futhark/TypeChecker/Consumption.hs +38/−18
• src/Language/Futhark/TypeChecker/Modules.hs +5/−4
• src/Language/Futhark/TypeChecker/Monad.hs +16/−13
• src/Language/Futhark/TypeChecker/Terms.hs +45/−26
• src/Language/Futhark/TypeChecker/Terms/DoLoop.hs +0/−288
• src/Language/Futhark/TypeChecker/Terms/Loop.hs +288/−0
• src/Language/Futhark/TypeChecker/Terms/Monad.hs +8/−5
• src/Language/Futhark/TypeChecker/Terms/Pat.hs +3/−1
• src/Language/Futhark/TypeChecker/Types.hs +14/−14
• src/Language/Futhark/TypeChecker/Unify.hs +46/−39
• unittests/Futhark/IR/Mem/IxFun/Alg.hs +11/−30
• unittests/Futhark/IR/Mem/IxFunTests.hs +100/−146
• unittests/Futhark/IR/Mem/IxFunWrapper.hs +8/−7
• unittests/Language/Futhark/SyntaxTests.hs +1/−1

@@ -60,6 +60,7 @@ :c:func:`futhark_context_new`.  A configuration object must not be freed before any context objects for which it is used.  The same configuration may *not* be used for multiple concurrent contexts.+Configuration objects are cheap to create and destroy.  .. c:struct:: futhark_context_config @@ -274,8 +275,9 @@ .. c:function:: int futhark_values_i32_1d(struct futhark_context *ctx, struct futhark_i32_1d *arr, int32_t *data)     Asynchronously copy data from the value into ``data``, which must-   be of sufficient size.  Multi-dimensional arrays are written in-   row-major form.+   point to free memory, allocated by the caller, with sufficient+   space to store the full array.  Multi-dimensional arrays are+   written in row-major form.  .. c:function:: const int64_t *futhark_shape_i32_1d(struct futhark_context *ctx, struct futhark_i32_1d *arr) @@ -439,8 +441,8 @@ GPU --- -The following API functions are available when using the ``opencl`` or-``cuda`` backends.+The following API functions are available when using the ``opencl``,+``cuda``, or ``hip`` backends.  .. c:function:: void futhark_context_config_set_device(struct futhark_context_config *cfg, const char *s) @@ -469,15 +471,15 @@    Set the default tile size used when executing kernels that have    been block tiled. -.. c:function:: void futhark_context_config_dump_program_to(struct futhark_context_config *cfg, const char *path)+.. c:function:: const char* futhark_context_config_get_program(struct futhark_context_config *cfg) -   During :c:func:`futhark_context_new`, dump the OpenCL or CUDA-   program source to the given file.+   Retrieve the embedded GPU program.  The context configuration keeps+   ownership, so don't free the string. -.. c:function:: void futhark_context_config_load_program_from(struct futhark_context_config *cfg, const char *path)+.. c:function:: void futhark_context_config_set_program(struct futhark_context_config *cfg, const char *program) -   During :c:func:`futhark_context_new`, read OpenCL or CUDA program-   source from the given file instead of using the embedded program.+   Instead of using the embedded GPU program, use the provided string,+   which is copied by this function.  OpenCL ------@@ -520,10 +522,9 @@    Add a build option to the OpenCL kernel compiler.  See the OpenCL    specification for `clBuildProgram` for available options. -.. c:function:: void futhark_context_config_dump_binary_to(struct futhark_context_config *cfg, const char *path)+.. c:function:: cl_program futhark_context_get_program(struct futhark_context_config *cfg) -   During :c:func:`futhark_context_new`, dump the compiled OpenCL-   binary to the given file.+   Retrieve the compiled OpenCL program.  .. c:function:: void futhark_context_config_load_binary_from(struct futhark_context_config *cfg, const char *path) @@ -547,7 +548,7 @@    Add a build option to the NVRTC compiler.  See the CUDA    documentation for ``nvrtcCompileProgram`` for available options. -.. c:function:: void futhark_context_config_dump_ptx_to(struct futhark_context_config *cfg, const char *path)+.. c:function:: void futhark_context_dump_ptx_to(struct futhark_context_config *cfg, const char *path)     During :c:func:`futhark_context_new`, dump the generated PTX code    to the given file.

@@ -45,16 +45,16 @@    :caption: Manual Pages    :maxdepth: 1 -   man/futhark.rst    man/futhark-autotune.rst    man/futhark-bench.rst    man/futhark-c.rst    man/futhark-cuda.rst    man/futhark-dataset.rst    man/futhark-doc.rst+   man/futhark-hip.rst+   man/futhark-ispc.rst    man/futhark-literate.rst    man/futhark-multicore.rst-   man/futhark-ispc.rst    man/futhark-opencl.rst    man/futhark-pkg.rst    man/futhark-pyopencl.rst@@ -62,5 +62,6 @@    man/futhark-repl.rst    man/futhark-run.rst    man/futhark-test.rst-   man/futhark-wasm.rst    man/futhark-wasm-multicore.rst+   man/futhark-wasm.rst+   man/futhark.rst

@@ -0,0 +1,128 @@+.. role:: ref(emphasis)++.. _futhark-hip(1):++==============+futhark-hip+==============++SYNOPSIS+========++futhark hip [options...] <program.fut>++DESCRIPTION+===========+++``futhark hip`` translates a Futhark program to C code invoking HIP+kernels, and either compiles that C code with a C compiler to an+executable binary program, or produces a ``.h`` and ``.c`` file that+can be linked with other code. The standard Futhark optimisation+pipeline is used.++``futhark hip`` uses ``-lhiprtc -lamdhip64`` to link.  If using+``--library``, you will need to do the same when linking the final+binary.  Although the HIP backend can be made to work on NVIDIA GPUs,+you are probably better off using the very similar+:ref:`futhark-cuda(1)`.++OPTIONS+=======++Accepts the same options as :ref:`futhark-c(1)`.++ENVIRONMENT VARIABLES+=====================++``CC``++  The C compiler used to compile the program.  Defaults to ``cc`` if+  unset.++``CFLAGS``++  Space-separated list of options passed to the C compiler.  Defaults+  to ``-O -std=c99`` if unset.++EXECUTABLE OPTIONS+==================++Generated executables accept the same options as those generated by+:ref:`futhark-c(1)`.  For commonality, the options use OpenCL+nomenclature ("group" instead of "thread block").++The following additional options are accepted.++-h, --help++  Print help text to standard output and exit.++--default-group-size=INT++  The default size of thread blocks that are launched.  Capped to the+  hardware limit if necessary.++--default-num-groups=INT++  The default number of thread blocks that are launched.++--default-threshold=INT++  The default parallelism threshold used for comparisons when+  selecting between code versions generated by incremental flattening.+  Intuitively, the amount of parallelism needed to saturate the GPU.++--default-tile-size=INT++  The default tile size used when performing two-dimensional tiling+  (the workgroup size will be the square of the tile size).++--dump-hip=FILE++  Don't run the program, but instead dump the embedded HIP kernels to+  the indicated file.  Useful if you want to see what is actually+  being executed.++--load-hip=FILE++  Instead of using the embedded HIP kernels, load them from the+  indicated file.++-n, --no-print-result++  Do not print the program result.++--build-option=OPT++  Add an additional build option to the string passed to the kernel+  compiler (HIPRTC).  Refer to the HIP documentation for which options+  are supported.  Be careful - some options can easily result in+  invalid results.++--param=ASSIGNMENT++  Set a tuning parameter to the given+  value. ``ASSIGNMENT`` must be of the form ``NAME=INT`` Use+  ``--print-params`` to see which names are available.++--print-params++  Print all tuning parameters that can be set with ``--param`` or+  ``--tuning``.++--tuning=FILE++  Read size=value assignments from the given file.++ENVIRONMENT+===========++If run without ``--library``, ``futhark hip`` will invoke a C+compiler to compile the generated C program into a binary.  This only+works if the C compiler can find the necessary HIP libraries.++SEE ALSO+========++:ref:`futhark(1)`

@@ -158,7 +158,7 @@ ~~~~~~~~~~~  The following options are supported by executables generated with the-GPU backends (``opencl``, ``pyopencl``, and ``cuda``).+GPU backends (``opencl``, ``pyopencl``, ``hip``, and ``cuda``).    ``-d/--device DEVICE`` 

@@ -1,6 +1,6 @@ cabal-version: 2.4 name:           futhark-version:        0.25.2+version:        0.25.3 synopsis:       An optimising compiler for a functional, array-oriented language.  description:    Futhark is a small programming language designed to be compiled to@@ -41,12 +41,16 @@     rts/c/context_prototypes.h     rts/c/backends/c.h     rts/c/backends/cuda.h+    rts/c/backends/hip.h     rts/c/backends/multicore.h     rts/c/backends/opencl.h     rts/c/lock.h+    rts/c/copy.h     rts/c/timing.h     rts/c/errors.h     rts/c/free_list.h+    rts/c/gpu.h+    rts/c/gpu_prototypes.h     rts/c/tuning.h     rts/c/values.h     rts/c/half.h@@ -58,10 +62,14 @@     rts/c/uniform.h     rts/c/util.h     rts/c/server.h+    rts/cuda/prelude.cu     rts/futhark-doc/style.css     rts/javascript/server.js     rts/javascript/values.js     rts/javascript/wrapperclasses.js+    rts/opencl/copy.cl+    rts/opencl/prelude.cl+    rts/opencl/transpose.cl     rts/python/tuning.py     rts/python/panic.py     rts/python/memory.py@@ -191,6 +199,7 @@       Futhark.CLI.Dev       Futhark.CLI.Doc       Futhark.CLI.Eval+      Futhark.CLI.HIP       Futhark.CLI.Literate       Futhark.CLI.LSP       Futhark.CLI.Main@@ -208,9 +217,8 @@       Futhark.CLI.Test       Futhark.CLI.WASM       Futhark.CodeGen.Backends.CCUDA-      Futhark.CodeGen.Backends.CCUDA.Boilerplate       Futhark.CodeGen.Backends.COpenCL-      Futhark.CodeGen.Backends.COpenCL.Boilerplate+      Futhark.CodeGen.Backends.HIP       Futhark.CodeGen.Backends.GenericC       Futhark.CodeGen.Backends.GenericC.CLI       Futhark.CodeGen.Backends.GenericC.Code@@ -225,6 +233,7 @@       Futhark.CodeGen.Backends.GenericPython.AST       Futhark.CodeGen.Backends.GenericPython.Options       Futhark.CodeGen.Backends.GenericWASM+      Futhark.CodeGen.Backends.GPU       Futhark.CodeGen.Backends.MulticoreC       Futhark.CodeGen.Backends.MulticoreC.Boilerplate       Futhark.CodeGen.Backends.MulticoreISPC@@ -237,6 +246,8 @@       Futhark.CodeGen.Backends.SequentialWASM       Futhark.CodeGen.Backends.SimpleRep       Futhark.CodeGen.RTS.C+      Futhark.CodeGen.RTS.CUDA+      Futhark.CodeGen.RTS.OpenCL       Futhark.CodeGen.RTS.Python       Futhark.CodeGen.RTS.JavaScript       Futhark.CodeGen.ImpCode@@ -248,7 +259,6 @@       Futhark.CodeGen.ImpGen.CUDA       Futhark.CodeGen.ImpGen.GPU       Futhark.CodeGen.ImpGen.GPU.Base-      Futhark.CodeGen.ImpGen.GPU.Copy       Futhark.CodeGen.ImpGen.GPU.Group       Futhark.CodeGen.ImpGen.GPU.SegHist       Futhark.CodeGen.ImpGen.GPU.SegMap@@ -257,7 +267,7 @@       Futhark.CodeGen.ImpGen.GPU.SegScan.SinglePass       Futhark.CodeGen.ImpGen.GPU.SegScan.TwoPass       Futhark.CodeGen.ImpGen.GPU.ToOpenCL-      Futhark.CodeGen.ImpGen.GPU.Transpose+      Futhark.CodeGen.ImpGen.HIP       Futhark.CodeGen.ImpGen.Multicore       Futhark.CodeGen.ImpGen.Multicore.Base       Futhark.CodeGen.ImpGen.Multicore.SegHist@@ -266,7 +276,6 @@       Futhark.CodeGen.ImpGen.Multicore.SegScan       Futhark.CodeGen.ImpGen.OpenCL       Futhark.CodeGen.ImpGen.Sequential-      Futhark.CodeGen.ImpGen.Transpose       Futhark.CodeGen.OpenCL.Heuristics       Futhark.Compiler       Futhark.Compiler.CLI@@ -449,7 +458,7 @@       Language.Futhark.TypeChecker.Modules       Language.Futhark.TypeChecker.Monad       Language.Futhark.TypeChecker.Terms-      Language.Futhark.TypeChecker.Terms.DoLoop+      Language.Futhark.TypeChecker.Terms.Loop       Language.Futhark.TypeChecker.Terms.Monad       Language.Futhark.TypeChecker.Terms.Pat       Language.Futhark.TypeChecker.Types@@ -472,7 +481,7 @@     , base16-bytestring     , binary >=0.8.3     , blaze-html >=0.9.0.1-    , bytestring >=0.10.8+    , bytestring >=0.11.2     , bytestring-to-vector >=0.3.0.1     , bmp >=1.2.6.3     , co-log-core

@@ -197,7 +197,7 @@  -- | Create a value for each point in a two-dimensional index space. ----- **Work:** *O(n ✕ W(f))*+-- **Work:** *O(n ✕ m ✕ W(f))* -- -- **Span:** *O(S(f))* def tabulate_2d 'a (n: i64) (m: i64) (f: i64 -> i64 -> a): *[n][m]a =@@ -205,7 +205,7 @@  -- | Create a value for each point in a three-dimensional index space. ----- **Work:** *O(n ✕ W(f))*+-- **Work:** *O(n ✕ m ✕ o ✕ W(f))* -- -- **Span:** *O(S(f))* def tabulate_3d 'a (n: i64) (m: i64) (o: i64) (f: i64 -> i64 -> i64 -> a): *[n][m][o]a =

@@ -1,82 +1,82 @@ // Start of atomics.h -inline int32_t atomic_xchg_i32_global(volatile __global int32_t *p, int32_t x);-inline int32_t atomic_xchg_i32_local(volatile __local int32_t *p, int32_t x);-inline int32_t atomic_cmpxchg_i32_global(volatile __global int32_t *p,-                                         int32_t cmp, int32_t val);-inline int32_t atomic_cmpxchg_i32_local(volatile __local int32_t *p,-                                        int32_t cmp, int32_t val);-inline int32_t atomic_add_i32_global(volatile __global int32_t *p, int32_t x);-inline int32_t atomic_add_i32_local(volatile __local int32_t *p, int32_t x);-inline float atomic_fadd_f32_global(volatile __global float *p, float x);-inline float atomic_fadd_f32_local(volatile __local float *p, float x);-inline int32_t atomic_smax_i32_global(volatile __global int32_t *p, int32_t x);-inline int32_t atomic_smax_i32_local(volatile __local int32_t *p, int32_t x);-inline int32_t atomic_smin_i32_global(volatile __global int32_t *p, int32_t x);-inline int32_t atomic_smin_i32_local(volatile __local int32_t *p, int32_t x);-inline uint32_t atomic_umax_i32_global(volatile __global uint32_t *p, uint32_t x);-inline uint32_t atomic_umax_i32_local(volatile __local uint32_t *p, uint32_t x);-inline uint32_t atomic_umin_i32_global(volatile __global uint32_t *p, uint32_t x);-inline uint32_t atomic_umin_i32_local(volatile __local uint32_t *p, uint32_t x);-inline int32_t atomic_and_i32_global(volatile __global int32_t *p, int32_t x);-inline int32_t atomic_and_i32_local(volatile __local int32_t *p, int32_t x);-inline int32_t atomic_or_i32_global(volatile __global int32_t *p, int32_t x);-inline int32_t atomic_or_i32_local(volatile __local int32_t *p, int32_t x);-inline int32_t atomic_xor_i32_global(volatile __global int32_t *p, int32_t x);-inline int32_t atomic_xor_i32_local(volatile __local int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_xchg_i32_global(volatile __global int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_xchg_i32_local(volatile __local int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_cmpxchg_i32_global(volatile __global int32_t *p,+                                                         int32_t cmp, int32_t val);+SCALAR_FUN_ATTR int32_t atomic_cmpxchg_i32_local(volatile __local int32_t *p,+                                                        int32_t cmp, int32_t val);+SCALAR_FUN_ATTR int32_t atomic_add_i32_global(volatile __global int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_add_i32_local(volatile __local int32_t *p, int32_t x);+SCALAR_FUN_ATTR float atomic_fadd_f32_global(volatile __global float *p, float x);+SCALAR_FUN_ATTR float atomic_fadd_f32_local(volatile __local float *p, float x);+SCALAR_FUN_ATTR int32_t atomic_smax_i32_global(volatile __global int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_smax_i32_local(volatile __local int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_smin_i32_global(volatile __global int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_smin_i32_local(volatile __local int32_t *p, int32_t x);+SCALAR_FUN_ATTR uint32_t atomic_umax_i32_global(volatile __global uint32_t *p, uint32_t x);+SCALAR_FUN_ATTR uint32_t atomic_umax_i32_local(volatile __local uint32_t *p, uint32_t x);+SCALAR_FUN_ATTR uint32_t atomic_umin_i32_global(volatile __global uint32_t *p, uint32_t x);+SCALAR_FUN_ATTR uint32_t atomic_umin_i32_local(volatile __local uint32_t *p, uint32_t x);+SCALAR_FUN_ATTR int32_t atomic_and_i32_global(volatile __global int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_and_i32_local(volatile __local int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_or_i32_global(volatile __global int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_or_i32_local(volatile __local int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_xor_i32_global(volatile __global int32_t *p, int32_t x);+SCALAR_FUN_ATTR int32_t atomic_xor_i32_local(volatile __local int32_t *p, int32_t x); -inline int32_t atomic_xchg_i32_global(volatile __global int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_xchg_i32_global(volatile __global int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicExch((int32_t*)p, x); #else   return atomic_xor(p, x); #endif } -inline int32_t atomic_xchg_i32_local(volatile __local int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_xchg_i32_local(volatile __local int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicExch((int32_t*)p, x); #else   return atomic_xor(p, x); #endif } -inline int32_t atomic_cmpxchg_i32_global(volatile __global int32_t *p,-                                         int32_t cmp, int32_t val) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_cmpxchg_i32_global(volatile __global int32_t *p,+                                                         int32_t cmp, int32_t val) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicCAS((int32_t*)p, cmp, val); #else   return atomic_cmpxchg(p, cmp, val); #endif } -inline int32_t atomic_cmpxchg_i32_local(volatile __local int32_t *p,-                                        int32_t cmp, int32_t val) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_cmpxchg_i32_local(volatile __local int32_t *p,+                                                        int32_t cmp, int32_t val) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicCAS((int32_t*)p, cmp, val); #else   return atomic_cmpxchg(p, cmp, val); #endif } -inline int32_t atomic_add_i32_global(volatile __global int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_add_i32_global(volatile __global int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicAdd((int32_t*)p, x); #else   return atomic_add(p, x); #endif } -inline int32_t atomic_add_i32_local(volatile __local int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_add_i32_local(volatile __local int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicAdd((int32_t*)p, x); #else   return atomic_add(p, x); #endif } -inline float atomic_fadd_f32_global(volatile __global float *p, float x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR float atomic_fadd_f32_global(volatile __global float *p, float x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicAdd((float*)p, x); #else   union { int32_t i; float f; } old;@@ -91,8 +91,8 @@ #endif } -inline float atomic_fadd_f32_local(volatile __local float *p, float x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR float atomic_fadd_f32_local(volatile __local float *p, float x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicAdd((float*)p, x); #else   union { int32_t i; float f; } old;@@ -107,112 +107,112 @@ #endif } -inline int32_t atomic_smax_i32_global(volatile __global int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_smax_i32_global(volatile __global int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMax((int32_t*)p, x); #else   return atomic_max(p, x); #endif } -inline int32_t atomic_smax_i32_local(volatile __local int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_smax_i32_local(volatile __local int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMax((int32_t*)p, x); #else   return atomic_max(p, x); #endif } -inline int32_t atomic_smin_i32_global(volatile __global int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_smin_i32_global(volatile __global int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMin((int32_t*)p, x); #else   return atomic_min(p, x); #endif } -inline int32_t atomic_smin_i32_local(volatile __local int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_smin_i32_local(volatile __local int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMin((int32_t*)p, x); #else   return atomic_min(p, x); #endif } -inline uint32_t atomic_umax_i32_global(volatile __global uint32_t *p, uint32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR uint32_t atomic_umax_i32_global(volatile __global uint32_t *p, uint32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMax((uint32_t*)p, x); #else   return atomic_max(p, x); #endif } -inline uint32_t atomic_umax_i32_local(volatile __local uint32_t *p, uint32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR uint32_t atomic_umax_i32_local(volatile __local uint32_t *p, uint32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMax((uint32_t*)p, x); #else   return atomic_max(p, x); #endif } -inline uint32_t atomic_umin_i32_global(volatile __global uint32_t *p, uint32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR uint32_t atomic_umin_i32_global(volatile __global uint32_t *p, uint32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMin((uint32_t*)p, x); #else   return atomic_min(p, x); #endif } -inline uint32_t atomic_umin_i32_local(volatile __local uint32_t *p, uint32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR uint32_t atomic_umin_i32_local(volatile __local uint32_t *p, uint32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMin((uint32_t*)p, x); #else   return atomic_min(p, x); #endif } -inline int32_t atomic_and_i32_global(volatile __global int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_and_i32_global(volatile __global int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicAnd((int32_t*)p, x); #else   return atomic_and(p, x); #endif } -inline int32_t atomic_and_i32_local(volatile __local int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_and_i32_local(volatile __local int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicAnd((int32_t*)p, x); #else   return atomic_and(p, x); #endif } -inline int32_t atomic_or_i32_global(volatile __global int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_or_i32_global(volatile __global int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicOr((int32_t*)p, x); #else   return atomic_or(p, x); #endif } -inline int32_t atomic_or_i32_local(volatile __local int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_or_i32_local(volatile __local int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicOr((int32_t*)p, x); #else   return atomic_or(p, x); #endif } -inline int32_t atomic_xor_i32_global(volatile __global int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_xor_i32_global(volatile __global int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicXor((int32_t*)p, x); #else   return atomic_xor(p, x); #endif } -inline int32_t atomic_xor_i32_local(volatile __local int32_t *p, int32_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int32_t atomic_xor_i32_local(volatile __local int32_t *p, int32_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicXor((int32_t*)p, x); #else   return atomic_xor(p, x);@@ -221,80 +221,80 @@  // Start of 64 bit atomics -#if defined(FUTHARK_CUDA) || defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP) || defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics) -inline int64_t atomic_xchg_i64_global(volatile __global int64_t *p, int64_t x);-inline int64_t atomic_xchg_i64_local(volatile __local int64_t *p, int64_t x);-inline int64_t atomic_cmpxchg_i64_global(volatile __global int64_t *p,-                                         int64_t cmp, int64_t val);-inline int64_t atomic_cmpxchg_i64_local(volatile __local int64_t *p,-                                        int64_t cmp, int64_t val);-inline int64_t atomic_add_i64_global(volatile __global int64_t *p, int64_t x);-inline int64_t atomic_add_i64_local(volatile __local int64_t *p, int64_t x);-inline int64_t atomic_smax_i64_global(volatile __global int64_t *p, int64_t x);-inline int64_t atomic_smax_i64_local(volatile __local int64_t *p, int64_t x);-inline int64_t atomic_smin_i64_global(volatile __global int64_t *p, int64_t x);-inline int64_t atomic_smin_i64_local(volatile __local int64_t *p, int64_t x);-inline uint64_t atomic_umax_i64_global(volatile __global uint64_t *p, uint64_t x);-inline uint64_t atomic_umax_i64_local(volatile __local uint64_t *p, uint64_t x);-inline uint64_t atomic_umin_i64_global(volatile __global uint64_t *p, uint64_t x);-uint64_t atomic_umin_i64_local(volatile __local uint64_t *p, uint64_t x);-inline int64_t atomic_and_i64_global(volatile __global int64_t *p, int64_t x);-inline int64_t atomic_and_i64_local(volatile __local int64_t *p, int64_t x);-inline int64_t atomic_or_i64_global(volatile __global int64_t *p, int64_t x);-inline int64_t atomic_or_i64_local(volatile __local int64_t *p, int64_t x);-inline int64_t atomic_xor_i64_global(volatile __global int64_t *p, int64_t x);-inline int64_t atomic_xor_i64_local(volatile __local int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_xchg_i64_global(volatile __global int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_xchg_i64_local(volatile __local int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_cmpxchg_i64_global(volatile __global int64_t *p,+                                                         int64_t cmp, int64_t val);+SCALAR_FUN_ATTR int64_t atomic_cmpxchg_i64_local(volatile __local int64_t *p,+                                                        int64_t cmp, int64_t val);+SCALAR_FUN_ATTR int64_t atomic_add_i64_global(volatile __global int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_add_i64_local(volatile __local int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_smax_i64_global(volatile __global int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_smax_i64_local(volatile __local int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_smin_i64_global(volatile __global int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_smin_i64_local(volatile __local int64_t *p, int64_t x);+SCALAR_FUN_ATTR uint64_t atomic_umax_i64_global(volatile __global uint64_t *p, uint64_t x);+SCALAR_FUN_ATTR uint64_t atomic_umax_i64_local(volatile __local uint64_t *p, uint64_t x);+SCALAR_FUN_ATTR uint64_t atomic_umin_i64_global(volatile __global uint64_t *p, uint64_t x);+SCALAR_FUN_ATTR uint64_t atomic_umin_i64_local(volatile __local uint64_t *p, uint64_t x);+SCALAR_FUN_ATTR int64_t atomic_and_i64_global(volatile __global int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_and_i64_local(volatile __local int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_or_i64_global(volatile __global int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_or_i64_local(volatile __local int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_xor_i64_global(volatile __global int64_t *p, int64_t x);+SCALAR_FUN_ATTR int64_t atomic_xor_i64_local(volatile __local int64_t *p, int64_t x);  #ifdef FUTHARK_F64_ENABLED-inline double atomic_fadd_f64_global(volatile __global double *p, double x);-inline double atomic_fadd_f64_local(volatile __local double *p, double x);+SCALAR_FUN_ATTR double atomic_fadd_f64_global(volatile __global double *p, double x);+SCALAR_FUN_ATTR double atomic_fadd_f64_local(volatile __local double *p, double x); #endif -inline int64_t atomic_xchg_i64_global(volatile __global int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_xchg_i64_global(volatile __global int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicExch((uint64_t*)p, x); #else   return atom_xor(p, x); #endif } -inline int64_t atomic_xchg_i64_local(volatile __local int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_xchg_i64_local(volatile __local int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicExch((uint64_t*)p, x); #else   return atom_xor(p, x); #endif } -inline int64_t atomic_cmpxchg_i64_global(volatile __global int64_t *p,-                                         int64_t cmp, int64_t val) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_cmpxchg_i64_global(volatile __global int64_t *p,+                                                         int64_t cmp, int64_t val) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicCAS((uint64_t*)p, cmp, val); #else   return atom_cmpxchg(p, cmp, val); #endif } -inline int64_t atomic_cmpxchg_i64_local(volatile __local int64_t *p,-                                        int64_t cmp, int64_t val) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_cmpxchg_i64_local(volatile __local int64_t *p,+                                                        int64_t cmp, int64_t val) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicCAS((uint64_t*)p, cmp, val); #else   return atom_cmpxchg(p, cmp, val); #endif } -inline int64_t atomic_add_i64_global(volatile __global int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_add_i64_global(volatile __global int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicAdd((uint64_t*)p, x); #else   return atom_add(p, x); #endif } -inline int64_t atomic_add_i64_local(volatile __local int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_add_i64_local(volatile __local int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicAdd((uint64_t*)p, x); #else   return atom_add(p, x);@@ -303,8 +303,8 @@  #ifdef FUTHARK_F64_ENABLED -inline double atomic_fadd_f64_global(volatile __global double *p, double x) {-#if defined(FUTHARK_CUDA) && __CUDA_ARCH__ >= 600+SCALAR_FUN_ATTR double atomic_fadd_f64_global(volatile __global double *p, double x) {+#if defined(FUTHARK_CUDA) && __CUDA_ARCH__ >= 600 || defined(FUTHARK_HIP)   return atomicAdd((double*)p, x); #else   union { int64_t i; double f; } old;@@ -319,8 +319,8 @@ #endif } -inline double atomic_fadd_f64_local(volatile __local double *p, double x) {-#if defined(FUTHARK_CUDA) && __CUDA_ARCH__ >= 600+SCALAR_FUN_ATTR double atomic_fadd_f64_local(volatile __local double *p, double x) {+#if defined(FUTHARK_CUDA) && __CUDA_ARCH__ >= 600 || defined(FUTHARK_HIP)   return atomicAdd((double*)p, x); #else   union { int64_t i; double f; } old;@@ -337,118 +337,154 @@  #endif -inline int64_t atomic_smax_i64_global(volatile __global int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_smax_i64_global(volatile __global int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA)   return atomicMax((int64_t*)p, x);+#elif defined(FUTHARK_HIP)+  // Currentely missing in HIP; probably a temporary oversight.+  int64_t old = *p, assumed;+  do {+    assumed = old;+    old = smax64(old, x);+    old = atomic_cmpxchg_i64_global((volatile __global int64_t*)p, assumed, old);+  } while (assumed != old);+  return old; #else   return atom_max(p, x); #endif } -inline int64_t atomic_smax_i64_local(volatile __local int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_smax_i64_local(volatile __local int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA)   return atomicMax((int64_t*)p, x);+#elif defined(FUTHARK_HIP)+  // Currentely missing in HIP; probably a temporary oversight.+  int64_t old = *p, assumed;+  do {+    assumed = old;+    old = smax64(old, x);+    old = atomic_cmpxchg_i64_local((volatile __local int64_t*)p, assumed, old);+  } while (assumed != old);+  return old; #else   return atom_max(p, x); #endif } -inline int64_t atomic_smin_i64_global(volatile __global int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_smin_i64_global(volatile __global int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA)   return atomicMin((int64_t*)p, x);+#elif defined(FUTHARK_HIP)+  // Currentely missing in HIP; probably a temporary oversight.+  int64_t old = *p, assumed;+  do {+    assumed = old;+    old = smin64(old, x);+    old = atomic_cmpxchg_i64_global((volatile __global int64_t*)p, assumed, old);+  } while (assumed != old);+  return old; #else   return atom_min(p, x); #endif } -inline int64_t atomic_smin_i64_local(volatile __local int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR int64_t atomic_smin_i64_local(volatile __local int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA)   return atomicMin((int64_t*)p, x);+#elif defined(FUTHARK_HIP)+  // Currentely missing in HIP; probably a temporary oversight.+  int64_t old = *p, assumed;+  do {+    assumed = old;+    old = smin64(old, x);+    old = atomic_cmpxchg_i64_local((volatile __local int64_t*)p, assumed, old);+  } while (assumed != old);+  return old; #else   return atom_min(p, x); #endif } -inline uint64_t atomic_umax_i64_global(volatile __global uint64_t *p, uint64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR uint64_t atomic_umax_i64_global(volatile __global uint64_t *p, uint64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMax((uint64_t*)p, x); #else   return atom_max(p, x); #endif } -inline uint64_t atomic_umax_i64_local(volatile __local uint64_t *p, uint64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR uint64_t atomic_umax_i64_local(volatile __local uint64_t *p, uint64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMax((uint64_t*)p, x); #else   return atom_max(p, x); #endif } -inline uint64_t atomic_umin_i64_global(volatile __global uint64_t *p, uint64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR uint64_t atomic_umin_i64_global(volatile __global uint64_t *p, uint64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMin((uint64_t*)p, x); #else   return atom_min(p, x); #endif } -inline uint64_t atomic_umin_i64_local(volatile __local uint64_t *p, uint64_t x) {-#ifdef FUTHARK_CUDA+SCALAR_FUN_ATTR uint64_t atomic_umin_i64_local(volatile __local uint64_t *p, uint64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)   return atomicMin((uint64_t*)p, x); #else   return atom_min(p, x); #endif } -inline int64_t atomic_and_i64_global(volatile __global int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA-  return atomicAnd((int64_t*)p, x);+SCALAR_FUN_ATTR int64_t atomic_and_i64_global(volatile __global int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)+  return atomicAnd((uint64_t*)p, x); #else   return atom_and(p, x); #endif } -inline int64_t atomic_and_i64_local(volatile __local int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA-  return atomicAnd((int64_t*)p, x);+SCALAR_FUN_ATTR int64_t atomic_and_i64_local(volatile __local int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)+  return atomicAnd((uint64_t*)p, x); #else   return atom_and(p, x); #endif } -inline int64_t atomic_or_i64_global(volatile __global int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA-  return atomicOr((int64_t*)p, x);+SCALAR_FUN_ATTR int64_t atomic_or_i64_global(volatile __global int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)+  return atomicOr((uint64_t*)p, x); #else   return atom_or(p, x); #endif } -inline int64_t atomic_or_i64_local(volatile __local int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA-  return atomicOr((int64_t*)p, x);+SCALAR_FUN_ATTR int64_t atomic_or_i64_local(volatile __local int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)+  return atomicOr((uint64_t*)p, x); #else   return atom_or(p, x); #endif } -inline int64_t atomic_xor_i64_global(volatile __global int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA-  return atomicXor((int64_t*)p, x);+SCALAR_FUN_ATTR int64_t atomic_xor_i64_global(volatile __global int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)+  return atomicXor((uint64_t*)p, x); #else   return atom_xor(p, x); #endif } -inline int64_t atomic_xor_i64_local(volatile __local int64_t *p, int64_t x) {-#ifdef FUTHARK_CUDA-  return atomicXor((int64_t*)p, x);+SCALAR_FUN_ATTR int64_t atomic_xor_i64_local(volatile __local int64_t *p, int64_t x) {+#if defined(FUTHARK_CUDA) || defined(FUTHARK_HIP)+  return atomicXor((uint64_t*)p, x); #else   return atom_xor(p, x); #endif } -#endif // defined(FUTHARK_CUDA) || defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)+#endif // defined(FUTHARK_CUDA) || defined(FUTHARK_HIP) || defined(cl_khr_int64_base_atomics) && defined(cl_khr_int64_extended_atomics)  // End of atomics.h

@@ -87,15 +87,13 @@   const char** tuning_param_classes;   // Uniform fields above. +  char* program;   int num_nvrtc_opts;   const char **nvrtc_opts;    const char *preferred_device;   int preferred_device_num; -  const char *dump_program_to;-  const char *load_program_from;-   const char *dump_ptx_to;   const char *load_ptx_from; @@ -115,10 +113,10 @@   cfg->nvrtc_opts = (const char**) malloc(sizeof(const char*));   cfg->nvrtc_opts[0] = NULL; +  cfg->program = strconcat(gpu_program);+   cfg->preferred_device_num = 0;   cfg->preferred_device = "";-  cfg->dump_program_to = NULL;-  cfg->load_program_from = NULL;    cfg->dump_ptx_to = NULL;   cfg->load_ptx_from = NULL;@@ -161,12 +159,12 @@   cfg->preferred_device_num = x; } -void futhark_context_config_dump_program_to(struct futhark_context_config *cfg, const char *path) {-  cfg->dump_program_to = path;+const char* futhark_context_config_get_program(struct futhark_context_config *cfg) {+  return cfg->program; } -void futhark_context_config_load_program_from(struct futhark_context_config *cfg, const char *path) {-  cfg->load_program_from = path;+void futhark_context_config_set_program(struct futhark_context_config *cfg, const char *s) {+  cfg->program = strdup(s); }  void futhark_context_config_dump_ptx_to(struct futhark_context_config *cfg, const char *path) {@@ -235,8 +233,7 @@ // A record of something that happened. struct profiling_record {   cudaEvent_t *events; // Points to two events.-  int *runs;-  int64_t *runtime;+  const char *name; };  struct futhark_context {@@ -272,13 +269,13 @@   CUmodule module;   CUstream stream; -  struct free_list cu_free_list;+  struct free_list gpu_free_list; -  size_t max_block_size;+  size_t max_group_size;   size_t max_grid_size;   size_t max_tile_size;   size_t max_threshold;-  size_t max_shared_memory;+  size_t max_local_memory;   size_t max_bespoke;    size_t lockstep_width;@@ -286,6 +283,8 @@   struct profiling_record *profiling_records;   int profiling_records_capacity;   int profiling_records_used;++  struct builtin_kernels* kernels; };  #define CU_DEV_ATTR(x) (CU_DEVICE_ATTRIBUTE_##x)@@ -331,13 +330,13 @@     name[sizeof(name) - 1] = 0;      if (cfg->logging) {-      fprintf(stderr, "Device #%d: name=\"%s\", compute capability=%d.%d\n",+      fprintf(ctx->log, "Device #%d: name=\"%s\", compute capability=%d.%d\n",               i, name, cc_major, cc_minor);     }      if (device_query(dev, COMPUTE_MODE) == CU_COMPUTEMODE_PROHIBITED) {       if (cfg->logging) {-        fprintf(stderr, "Device #%d is compute-prohibited, ignoring\n", i);+        fprintf(ctx->log, "Device #%d is compute-prohibited, ignoring\n", i);       }       continue;     }@@ -360,31 +359,13 @@   if (chosen == -1) { return 1; }    if (cfg->logging) {-    fprintf(stderr, "Using device #%d\n", chosen);+    fprintf(ctx->log, "Using device #%d\n", chosen);   }    CUDA_SUCCEED_FATAL(cuDeviceGet(&ctx->dev, chosen));   return 0; } -static char *concat_fragments(const char *src_fragments[]) {-  size_t src_len = 0;-  const char **p;--  for (p = src_fragments; *p; p++) {-    src_len += strlen(*p);-  }--  char *src = (char*) malloc(src_len + 1);-  size_t n = 0;-  for (p = src_fragments; *p; p++) {-    strcpy(src + n, *p);-    n += strlen(*p);-  }--  return src;-}- static const char *cuda_nvrtc_get_arch(CUdevice dev) {   static struct {     int major;@@ -467,13 +448,13 @@   }   opts[i++] = msgprintf("-D%s=%d",                         "max_group_size",-                        (int)ctx->max_block_size);+                        (int)ctx->max_group_size);   for (int j = 0; j < cfg->num_tuning_params; j++) {     opts[i++] = msgprintf("-D%s=%zu", cfg->tuning_param_vars[j],                           cfg->tuning_params[j]);   }   opts[i++] = msgprintf("-DLOCKSTEP_WIDTH=%zu", ctx->lockstep_width);-  opts[i++] = msgprintf("-DMAX_THREADS_PER_BLOCK=%zu", ctx->max_block_size);+  opts[i++] = msgprintf("-DMAX_THREADS_PER_BLOCK=%zu", ctx->max_group_size);    // Time for the best lines of the code in the entire compiler.   if (getenv("CUDA_HOME") != NULL) {@@ -492,6 +473,10 @@     opts[i++] = strdup(extra_opts[j]);   } +  opts[i++] = msgprintf("-DTR_BLOCK_DIM=%d", TR_BLOCK_DIM);+  opts[i++] = msgprintf("-DTR_TILE_DIM=%d", TR_TILE_DIM);+  opts[i++] = msgprintf("-DTR_ELEMS_PER_THREAD=%d", TR_ELEMS_PER_THREAD);+   *n_opts = i;   *opts_out = opts; }@@ -557,13 +542,13 @@ static void cuda_size_setup(struct futhark_context *ctx) {   struct futhark_context_config *cfg = ctx->cfg;-  if (cfg->default_block_size > ctx->max_block_size) {+  if (cfg->default_block_size > ctx->max_group_size) {     if (cfg->default_block_size_changed) {       fprintf(stderr,               "Note: Device limits default block size to %zu (down from %zu).\n",-              ctx->max_block_size, cfg->default_block_size);+              ctx->max_group_size, cfg->default_block_size);     }-    cfg->default_block_size = ctx->max_block_size;+    cfg->default_block_size = ctx->max_group_size;   }   if (cfg->default_grid_size > ctx->max_grid_size) {     if (cfg->default_grid_size_changed) {@@ -596,7 +581,7 @@     int64_t max_value = 0, default_value = 0;      if (strstr(size_class, "group_size") == size_class) {-      max_value = ctx->max_block_size;+      max_value = ctx->max_group_size;       default_value = cfg->default_block_size;     } else if (strstr(size_class, "num_groups") == size_class) {       max_value = ctx->max_grid_size;@@ -631,31 +616,16 @@ }  static char* cuda_module_setup(struct futhark_context *ctx,-                               const char *src_fragments[],+                               const char *src,                                const char *extra_opts[],                                const char* cache_fname) {-  char *ptx = NULL, *src = NULL;+  char *ptx = NULL;   struct futhark_context_config *cfg = ctx->cfg; -  if (cfg->load_program_from == NULL) {-    src = concat_fragments(src_fragments);-  } else {-    src = slurp_file(cfg->load_program_from, NULL);-  }-   if (cfg->load_ptx_from) {-    if (cfg->load_program_from != NULL) {-      fprintf(stderr,-              "WARNING: Using PTX from %s instead of C code from %s\n",-              cfg->load_ptx_from, cfg->load_program_from);-    }     ptx = slurp_file(cfg->load_ptx_from, NULL);   } -  if (cfg->dump_program_to != NULL) {-    dump_file(cfg->dump_program_to, src, strlen(src));-  }-   char **opts;   size_t n_opts;   cuda_nvrtc_mk_build_options(ctx, extra_opts, &opts, &n_opts);@@ -695,7 +665,6 @@   if (ptx == NULL) {     char* problem = cuda_nvrtc_build(src, (const char**)opts, n_opts, &ptx);     if (problem != NULL) {-      free(src);       return problem;     }   }@@ -722,18 +691,15 @@     free((char *)opts[i]);   }   free(opts);-   free(ptx);-  if (src != NULL) {-    free(src);-  }    return NULL; }  // Count up the runtime all the profiling_records that occured during execution. // Also clears the buffer of profiling_records.-static CUresult cuda_tally_profiling_records(struct futhark_context *ctx) {+static CUresult tally_profiling_records(struct futhark_context *ctx,+                                        struct cost_centres* ccs) {   CUresult err;   for (int i = 0; i < ctx->profiling_records_used; i++) {     struct profiling_record record = ctx->profiling_records[i];@@ -743,9 +709,15 @@       return err;     } -    // CUDA provides milisecond resolution, but we want microseconds.-    *record.runs += 1;-    *record.runtime += ms*1000;+    if (ccs) {+      // CUDA provides milisecond resolution, but we want microseconds.+      struct cost_centre c = {+        .name = record.name,+        .runs = 1,+        .runtime = ms*1000+      };+      cost_centres_add(ccs, c);+    }      if ((err = cuEventDestroy(record.events[0])) != CUDA_SUCCESS) {       return err;@@ -763,7 +735,7 @@ }  // Returns pointer to two events.-static cudaEvent_t* cuda_get_events(struct futhark_context *ctx, int *runs, int64_t *runtime) {+static cudaEvent_t* cuda_get_events(struct futhark_context *ctx, const char* name) {   if (ctx->profiling_records_used == ctx->profiling_records_capacity) {     ctx->profiling_records_capacity *= 2;     ctx->profiling_records =@@ -775,79 +747,11 @@   cudaEventCreate(&events[0]);   cudaEventCreate(&events[1]);   ctx->profiling_records[ctx->profiling_records_used].events = events;-  ctx->profiling_records[ctx->profiling_records_used].runs = runs;-  ctx->profiling_records[ctx->profiling_records_used].runtime = runtime;+  ctx->profiling_records[ctx->profiling_records_used].name = name;   ctx->profiling_records_used++;   return events; } -static CUresult cuda_alloc(struct futhark_context *ctx, FILE *log,-                           size_t min_size, const char *tag,-                           CUdeviceptr *mem_out, size_t *size_out) {-  if (min_size < sizeof(int)) {-    min_size = sizeof(int);-  }--  if (free_list_find(&ctx->cu_free_list, min_size, tag, size_out, (fl_mem*)mem_out) == 0) {-    if (*size_out >= min_size) {-      if (ctx->cfg->debugging) {-        fprintf(log, "No need to allocate: Found a block in the free list.\n");-      }-      return CUDA_SUCCESS;-    } else {-      if (ctx->cfg->debugging) {-        fprintf(log, "Found a free block, but it was too small.\n");-      }--      CUresult res = cuMemFree(*mem_out);-      if (res != CUDA_SUCCESS) {-        return res;-      }-    }-  }--  *size_out = min_size;--  if (ctx->cfg->debugging) {-    fprintf(log, "Actually allocating the desired block.\n");-  }--  CUresult res = cuMemAlloc(mem_out, min_size);-  while (res == CUDA_ERROR_OUT_OF_MEMORY) {-    CUdeviceptr mem;-    if (free_list_first(&ctx->cu_free_list, (fl_mem*)&mem) == 0) {-      res = cuMemFree(mem);-      if (res != CUDA_SUCCESS) {-        return res;-      }-    } else {-      break;-    }-    res = cuMemAlloc(mem_out, min_size);-  }--  return res;-}--static CUresult cuda_free(struct futhark_context *ctx,-                          CUdeviceptr mem, size_t size, const char *tag) {-  free_list_insert(&ctx->cu_free_list, size, (fl_mem)mem, tag);-  return CUDA_SUCCESS;-}--static CUresult cuda_free_all(struct futhark_context *ctx) {-  CUdeviceptr mem;-  free_list_pack(&ctx->cu_free_list);-  while (free_list_first(&ctx->cu_free_list, (fl_mem*)&mem) == 0) {-    CUresult res = cuMemFree(mem);-    if (res != CUDA_SUCCESS) {-      return res;-    }-  }--  return CUDA_SUCCESS;-}- int futhark_context_sync(struct futhark_context* ctx) {   CUDA_SUCCEED_OR_RETURN(cuCtxPushCurrent(ctx->cu_ctx));   CUDA_SUCCEED_OR_RETURN(cuCtxSynchronize());@@ -880,10 +784,14 @@       return FUTHARK_PROGRAM_ERROR;     }   }+   CUDA_SUCCEED_OR_RETURN(cuCtxPopCurrent(&ctx->cu_ctx));   return 0; } +struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx);+void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels);+ int backend_context_setup(struct futhark_context* ctx) {   ctx->profiling_records_capacity = 200;   ctx->profiling_records_used = 0;@@ -902,18 +810,18 @@   }   CUDA_SUCCEED_FATAL(cuCtxCreate(&ctx->cu_ctx, 0, ctx->dev)); -  free_list_init(&ctx->cu_free_list);+  free_list_init(&ctx->gpu_free_list); -  ctx->max_shared_memory = device_query(ctx->dev, MAX_SHARED_MEMORY_PER_BLOCK);-  ctx->max_block_size = device_query(ctx->dev, MAX_THREADS_PER_BLOCK);+  ctx->max_local_memory = device_query(ctx->dev, MAX_SHARED_MEMORY_PER_BLOCK);+  ctx->max_group_size = device_query(ctx->dev, MAX_THREADS_PER_BLOCK);   ctx->max_grid_size = device_query(ctx->dev, MAX_GRID_DIM_X);-  ctx->max_tile_size = sqrt(ctx->max_block_size);+  ctx->max_tile_size = sqrt(ctx->max_group_size);   ctx->max_threshold = 0;   ctx->max_bespoke = 0;   ctx->lockstep_width = device_query(ctx->dev, WARP_SIZE);   CUDA_SUCCEED_FATAL(cuStreamCreate(&ctx->stream, CU_STREAM_DEFAULT));   cuda_size_setup(ctx);-  ctx->error = cuda_module_setup(ctx, cuda_program,+  ctx->error = cuda_module_setup(ctx, ctx->cfg->program,                                  ctx->cfg->nvrtc_opts, ctx->cfg->cache_fname);    if (ctx->error != NULL) {@@ -925,18 +833,207 @@   CUDA_SUCCEED_FATAL(cuMemcpyHtoD(ctx->global_failure, &no_error, sizeof(no_error)));   // The +1 is to avoid zero-byte allocations.   CUDA_SUCCEED_FATAL(cuMemAlloc(&ctx->global_failure_args, sizeof(int64_t)*(max_failure_args+1)));++  if ((ctx->kernels = init_builtin_kernels(ctx)) == NULL) {+    return 1;+  }+   return 0; }  void backend_context_teardown(struct futhark_context* ctx) {+  free_builtin_kernels(ctx, ctx->kernels);   cuMemFree(ctx->global_failure);   cuMemFree(ctx->global_failure_args);-  CUDA_SUCCEED_FATAL(cuda_free_all(ctx));-  (void)cuda_tally_profiling_records(ctx);+  CUDA_SUCCEED_FATAL(gpu_free_all(ctx));+  (void)tally_profiling_records(ctx, NULL);   free(ctx->profiling_records);   CUDA_SUCCEED_FATAL(cuStreamDestroy(ctx->stream));   CUDA_SUCCEED_FATAL(cuModuleUnload(ctx->module));   CUDA_SUCCEED_FATAL(cuCtxDestroy(ctx->cu_ctx));+}++// GPU ABSTRACTION LAYER++// Types.++typedef CUfunction gpu_kernel;+typedef CUdeviceptr gpu_mem;++static void gpu_create_kernel(struct futhark_context *ctx,+                              gpu_kernel* kernel,+                              const char* name) {+  if (ctx->debugging) {+    fprintf(ctx->log, "Creating kernel %s.\n", name);+  }+  CUDA_SUCCEED_FATAL(cuModuleGetFunction(kernel, ctx->module, name));+}++static void gpu_free_kernel(struct futhark_context *ctx,+                            gpu_kernel kernel) {+  (void)ctx;+  (void)kernel;+}++static int gpu_scalar_to_device(struct futhark_context* ctx,+                                gpu_mem dst, size_t offset, size_t size,+                                void *src) {+  CUevent *pevents = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    pevents = cuda_get_events(ctx, "copy_scalar_to_dev");+    CUDA_SUCCEED_FATAL(cuEventRecord(pevents[0], ctx->stream));+  }+  CUDA_SUCCEED_OR_RETURN(cuMemcpyHtoD(dst + offset, src, size));+  if (pevents != NULL) {+    CUDA_SUCCEED_FATAL(cuEventRecord(pevents[1], ctx->stream));+  }+  return FUTHARK_SUCCESS;+}++static int gpu_scalar_from_device(struct futhark_context* ctx,+                                  void *dst,+                                  gpu_mem src, size_t offset, size_t size) {+  CUevent *pevents = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    pevents = cuda_get_events(ctx, "copy_scalar_from_dev");+    CUDA_SUCCEED_FATAL(cuEventRecord(pevents[0], ctx->stream));+  }+  CUDA_SUCCEED_OR_RETURN(cuMemcpyDtoH(dst, src + offset, size));+  if (pevents != NULL) {+    CUDA_SUCCEED_FATAL(cuEventRecord(pevents[1], ctx->stream));+  }+  return FUTHARK_SUCCESS;+}++static int gpu_memcpy(struct futhark_context* ctx,+                      gpu_mem dst, int64_t dst_offset,+                      gpu_mem src, int64_t src_offset,+                      int64_t nbytes) {+  CUevent *pevents = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    pevents = cuda_get_events(ctx, "copy_dev_to_dev");+    CUDA_SUCCEED_FATAL(cuEventRecord(pevents[0], ctx->stream));+  }+  CUDA_SUCCEED_OR_RETURN(cuMemcpy(dst+dst_offset, src+src_offset, nbytes));+  if (pevents != NULL) {+    CUDA_SUCCEED_FATAL(cuEventRecord(pevents[1], ctx->stream));+  }+  return FUTHARK_SUCCESS;+}++static int memcpy_host2gpu(struct futhark_context* ctx, bool sync,+                           gpu_mem dst, int64_t dst_offset,+                           const unsigned char* src, int64_t src_offset,+                           int64_t nbytes) {+  if (nbytes > 0) {+    CUevent* pevents = NULL;+    if (ctx->profiling && !ctx->profiling_paused) {+      pevents = cuda_get_events(ctx, "copy_host_to_dev");+      CUDA_SUCCEED_FATAL(cuEventRecord(pevents[0], ctx->stream));+    }+    if (sync) {+      CUDA_SUCCEED_OR_RETURN+        (cuMemcpyHtoD(dst + dst_offset, src + src_offset, nbytes));+    } else {+      CUDA_SUCCEED_OR_RETURN+        (cuMemcpyHtoDAsync(dst + dst_offset, src + src_offset, nbytes, ctx->stream));+    }+    if (pevents != NULL) {+      CUDA_SUCCEED_FATAL(cuEventRecord(pevents[1], ctx->stream));+    }+  }+  return FUTHARK_SUCCESS;+}++static int memcpy_gpu2host(struct futhark_context* ctx, bool sync,+                           unsigned char* dst, int64_t dst_offset,+                           gpu_mem src, int64_t src_offset,+                           int64_t nbytes) {+  if (nbytes > 0) {+    CUevent* pevents = NULL;+    if (ctx->profiling && !ctx->profiling_paused) {+      pevents = cuda_get_events(ctx, "copy_dev_to_host");+      CUDA_SUCCEED_FATAL(cuEventRecord(pevents[0], ctx->stream));+    }+    if (sync) {+      CUDA_SUCCEED_OR_RETURN+        (cuMemcpyDtoH(dst + dst_offset, src + src_offset, nbytes));+    } else {+      CUDA_SUCCEED_OR_RETURN+        (cuMemcpyDtoHAsync(dst + dst_offset, src + src_offset, nbytes, ctx->stream));+    }+    if (sync &&+        ctx->failure_is_an_option &&+        futhark_context_sync(ctx) != 0) {+      return 1;+    }+  }+  return FUTHARK_SUCCESS;+}++static int gpu_launch_kernel(struct futhark_context* ctx,+                             gpu_kernel kernel, const char *name,+                             const int32_t grid[3],+                             const int32_t block[3],+                             unsigned int local_mem_bytes,+                             int num_args,+                             void* args[num_args],+                             size_t args_sizes[num_args]) {+  (void) args_sizes;+  int64_t time_start = 0, time_end = 0;+  if (ctx->logging) {+    fprintf(ctx->log,+            "Launching kernel %s with\n"+            "  grid=(%d,%d,%d)\n"+            "  block=(%d,%d,%d)\n"+            "  local memory=%d\n",+            name,+            grid[0], grid[1], grid[2],+            block[0], block[1], block[2],+            local_mem_bytes);+    time_start = get_wall_time();+  }++  CUevent *pevents = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    pevents = cuda_get_events(ctx, name);+    CUDA_SUCCEED_FATAL(cuEventRecord(pevents[0], ctx->stream));+  }++  CUDA_SUCCEED_OR_RETURN+    (cuLaunchKernel(kernel,+                    grid[0], grid[1], grid[2],+                    block[0], block[1], block[2],+                    local_mem_bytes, ctx->stream,+                    args, NULL));++  if (pevents != NULL) {+    CUDA_SUCCEED_FATAL(cuEventRecord(pevents[1], ctx->stream));+  }++  if (ctx->debugging) {+    CUDA_SUCCEED_FATAL(cuCtxSynchronize());+    time_end = get_wall_time();+    long int time_diff = time_end - time_start;+    fprintf(ctx->log, "  runtime: %ldus\n\n", time_diff);+  }++  return FUTHARK_SUCCESS;+}++static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out) {+  CUresult res = cuMemAlloc(mem_out, size);+  if (res == CUDA_ERROR_OUT_OF_MEMORY) {+    return FUTHARK_OUT_OF_MEMORY;+  }+  CUDA_SUCCEED_OR_RETURN(res);+  return FUTHARK_SUCCESS;+}++static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem) {+  (void)ctx;+  CUDA_SUCCEED_OR_RETURN(cuMemFree(mem));+  return FUTHARK_SUCCESS; }  // End of backends/cuda.h.

@@ -0,0 +1,911 @@+// Start of backends/hip.h.++// Forward declarations.+// Invoked by setup_opencl() after the platform and device has been+// found, but before the program is loaded.  Its intended use is to+// tune constants based on the selected platform and device.+static void set_tuning_params(struct futhark_context* ctx);+static char* get_failure_msg(int failure_idx, int64_t args[]);++#define HIP_SUCCEED_FATAL(x) hip_api_succeed_fatal(x, #x, __FILE__, __LINE__)+#define HIP_SUCCEED_NONFATAL(x) hip_api_succeed_nonfatal(x, #x, __FILE__, __LINE__)+#define HIPRTC_SUCCEED_FATAL(x) hiprtc_api_succeed_fatal(x, #x, __FILE__, __LINE__)+#define HIPRTC_SUCCEED_NONFATAL(x) hiprtc_api_succeed_nonfatal(x, #x, __FILE__, __LINE__)+// Take care not to override an existing error.+#define HIP_SUCCEED_OR_RETURN(e) {             \+    char *serror = HIP_SUCCEED_NONFATAL(e);    \+    if (serror) {                               \+      if (!ctx->error) {                        \+        ctx->error = serror;                    \+        return bad;                             \+      } else {                                  \+        free(serror);                           \+      }                                         \+    }                                           \+  }++// HIP_SUCCEED_OR_RETURN returns the value of the variable 'bad' in+// scope.  By default, it will be this one.  Create a local variable+// of some other type if needed.  This is a bit of a hack, but it+// saves effort in the code generator.+static const int bad = 1;++static inline void hip_api_succeed_fatal(hipError_t res, const char *call,+                                         const char *file, int line) {+  if (res != hipSuccess) {+    const char *err_str = hipGetErrorString(res);+    if (err_str == NULL) { err_str = "Unknown"; }+    futhark_panic(-1, "%s:%d: HIP call\n  %s\nfailed with error code %d (%s)\n",+                  file, line, call, res, err_str);+  }+}++static char* hip_api_succeed_nonfatal(hipError_t res, const char *call,+                                      const char *file, int line) {+  if (res != hipSuccess) {+    const char *err_str = hipGetErrorString(res);+    if (err_str == NULL) { err_str = "Unknown"; }+    return msgprintf("%s:%d: HIP call\n  %s\nfailed with error code %d (%s)\n",+                     file, line, call, res, err_str);+  } else {+    return NULL;+  }+}++static inline void hiprtc_api_succeed_fatal(hiprtcResult res, const char *call,+                                           const char *file, int line) {+  if (res != HIPRTC_SUCCESS) {+    const char *err_str = hiprtcGetErrorString(res);+    futhark_panic(-1, "%s:%d: HIPRTC call\n  %s\nfailed with error code %d (%s)\n",+                  file, line, call, res, err_str);+  }+}++static char* hiprtc_api_succeed_nonfatal(hiprtcResult res, const char *call,+                                        const char *file, int line) {+  if (res != HIPRTC_SUCCESS) {+    const char *err_str = hiprtcGetErrorString(res);+    return msgprintf("%s:%d: HIPRTC call\n  %s\nfailed with error code %d (%s)\n",+                     file, line, call, res, err_str);+  } else {+    return NULL;+  }+}++struct futhark_context_config {+  int in_use;+  int debugging;+  int profiling;+  int logging;+  const char *cache_fname;+  int num_tuning_params;+  int64_t *tuning_params;+  const char** tuning_param_names;+  const char** tuning_param_vars;+  const char** tuning_param_classes;+  // Uniform fields above.++  char* program;+  int num_build_opts;+  const char **build_opts;++  const char *preferred_device;+  int preferred_device_num;++  size_t default_block_size;+  size_t default_grid_size;+  size_t default_tile_size;+  size_t default_reg_tile_size;+  size_t default_threshold;++  int default_block_size_changed;+  int default_grid_size_changed;+  int default_tile_size_changed;+};++static void backend_context_config_setup(struct futhark_context_config *cfg) {+  cfg->num_build_opts = 0;+  cfg->build_opts = (const char**) malloc(sizeof(const char*));+  cfg->build_opts[0] = NULL;+  cfg->preferred_device_num = 0;+  cfg->preferred_device = "";+  cfg->program = strconcat(gpu_program);++  cfg->default_block_size = 256;+  cfg->default_grid_size = 0; // Set properly later.+  cfg->default_tile_size = 32;+  cfg->default_reg_tile_size = 2;+  cfg->default_threshold = 32*1024;++  cfg->default_block_size_changed = 0;+  cfg->default_grid_size_changed = 0;+  cfg->default_tile_size_changed = 0;+}++static void backend_context_config_teardown(struct futhark_context_config* cfg) {+  free(cfg->build_opts);+  free(cfg->program);+}++void futhark_context_config_add_build_option(struct futhark_context_config *cfg, const char *opt) {+  cfg->build_opts[cfg->num_build_opts] = opt;+  cfg->num_build_opts++;+  cfg->build_opts = (const char **) realloc(cfg->build_opts, (cfg->num_build_opts + 1) * sizeof(const char *));+  cfg->build_opts[cfg->num_build_opts] = NULL;+}++void futhark_context_config_set_device(struct futhark_context_config *cfg, const char *s) {+  int x = 0;+  if (*s == '#') {+    s++;+    while (isdigit(*s)) {+      x = x * 10 + (*s++)-'0';+    }+    // Skip trailing spaces.+    while (isspace(*s)) {+      s++;+    }+  }+  cfg->preferred_device = s;+  cfg->preferred_device_num = x;+}+++const char* futhark_context_config_get_program(struct futhark_context_config *cfg) {+  return cfg->program;+}++void futhark_context_config_set_program(struct futhark_context_config *cfg, const char *s) {+  cfg->program = strdup(s);+}++void futhark_context_config_set_default_group_size(struct futhark_context_config *cfg, int size) {+  cfg->default_block_size = size;+  cfg->default_block_size_changed = 1;+}++void futhark_context_config_set_default_num_groups(struct futhark_context_config *cfg, int num) {+  cfg->default_grid_size = num;+  cfg->default_grid_size_changed = 1;+}++void futhark_context_config_set_default_tile_size(struct futhark_context_config *cfg, int size) {+  cfg->default_tile_size = size;+  cfg->default_tile_size_changed = 1;+}++void futhark_context_config_set_default_reg_tile_size(struct futhark_context_config *cfg, int size) {+  cfg->default_reg_tile_size = size;+}++void futhark_context_config_set_default_threshold(struct futhark_context_config *cfg, int size) {+  cfg->default_threshold = size;+}++int futhark_context_config_set_tuning_param(struct futhark_context_config *cfg,+                                            const char *param_name,+                                            size_t new_value) {+  for (int i = 0; i < cfg->num_tuning_params; i++) {+    if (strcmp(param_name, cfg->tuning_param_names[i]) == 0) {+      cfg->tuning_params[i] = new_value;+      return 0;+    }+  }+  if (strcmp(param_name, "default_group_size") == 0) {+    cfg->default_block_size = new_value;+    return 0;+  }+  if (strcmp(param_name, "default_num_groups") == 0) {+    cfg->default_grid_size = new_value;+    return 0;+  }+  if (strcmp(param_name, "default_threshold") == 0) {+    cfg->default_threshold = new_value;+    return 0;+  }+  if (strcmp(param_name, "default_tile_size") == 0) {+    cfg->default_tile_size = new_value;+    return 0;+  }+  if (strcmp(param_name, "default_reg_tile_size") == 0) {+    cfg->default_reg_tile_size = new_value;+    return 0;+  }+  return 1;+}++// A record of something that happened.+struct profiling_record {+  hipEvent_t *events; // Points to two events.+  const char *name;+};++struct futhark_context {+  struct futhark_context_config* cfg;+  int detail_memory;+  int debugging;+  int profiling;+  int profiling_paused;+  int logging;+  lock_t lock;+  char *error;+  lock_t error_lock;+  FILE *log;+  struct constants *constants;+  struct free_list free_list;+  int64_t peak_mem_usage_default;+  int64_t cur_mem_usage_default;+  // Uniform fields above.++  void* global_failure;+  void* global_failure_args;+  struct tuning_params tuning_params;+  // True if a potentially failing kernel has been enqueued.+  int32_t failure_is_an_option;+  int total_runs;+  long int total_runtime;+  int64_t peak_mem_usage_device;+  int64_t cur_mem_usage_device;+  struct program* program;++  hipDevice_t dev;+  int dev_id;+  hipModule_t module;+  hipStream_t stream;++  struct free_list gpu_free_list;++  size_t max_group_size;+  size_t max_grid_size;+  size_t max_tile_size;+  size_t max_threshold;+  size_t max_local_memory;+  size_t max_bespoke;++  size_t lockstep_width;++  struct profiling_record *profiling_records;+  int profiling_records_capacity;+  int profiling_records_used;++  struct builtin_kernels* kernels;+};++static int device_query(int dev_id, hipDeviceAttribute_t attr) {+  int val;+  HIP_SUCCEED_FATAL(hipDeviceGetAttribute(&val, attr, dev_id));+  return val;+}++static int function_query(hipFunction_t f, hipFunction_attribute attr) {+  int val;+  HIP_SUCCEED_FATAL(hipFuncGetAttribute(&val, attr, f));+  return val;+}++static int hip_device_setup(struct futhark_context *ctx) {+  struct futhark_context_config *cfg = ctx->cfg;+  int count, chosen = -1;+  hipDevice_t dev;++  HIP_SUCCEED_FATAL(hipGetDeviceCount(&count));+  if (count == 0) { return 1; }++  int num_device_matches = 0;++  for (int i = 0; i < count; i++) {+    hipDeviceProp_t prop;+    hipGetDeviceProperties(&prop, i);++    if (cfg->logging) {+      fprintf(ctx->log, "Device #%d: name=\"%s\"\n", i, prop.name);+    }++    if (strstr(prop.name, cfg->preferred_device) != NULL &&+        num_device_matches++ == cfg->preferred_device_num) {+      chosen = i;+      break;+    }+  }++  if (chosen == -1) { return 1; }++  if (cfg->logging) {+    fprintf(ctx->log, "Using device #%d\n", chosen);+  }++  ctx->dev_id = chosen;+  HIP_SUCCEED_FATAL(hipDeviceGet(&ctx->dev, ctx->dev_id));+  return 0;+}++static void hip_load_code_from_cache(struct futhark_context_config *cfg,+                                     const char *src,+                                     const char *opts[], size_t n_opts,+                                     struct cache_hash *h, const char *cache_fname,+                                     char **code, size_t *code_size) {+  if (cfg->logging) {+    fprintf(stderr, "Restoring cache from from %s...\n", cache_fname);+  }+  cache_hash_init(h);+  for (size_t i = 0; i < n_opts; i++) {+    cache_hash(h, opts[i], strlen(opts[i]));+  }+  cache_hash(h, src, strlen(src));+  errno = 0;+  if (cache_restore(cache_fname, h, (unsigned char**)code, code_size) != 0) {+    if (cfg->logging) {+      fprintf(stderr, "Failed to restore cache (errno: %s)\n", strerror(errno));+    }+  }+}++static void hip_size_setup(struct futhark_context *ctx) {+  struct futhark_context_config *cfg = ctx->cfg;+  if (cfg->default_block_size > ctx->max_group_size) {+    if (cfg->default_block_size_changed) {+      fprintf(stderr,+              "Note: Device limits default block size to %zu (down from %zu).\n",+              ctx->max_group_size, cfg->default_block_size);+    }+    cfg->default_block_size = ctx->max_group_size;+  }+  if (cfg->default_grid_size > ctx->max_grid_size) {+    if (cfg->default_grid_size_changed) {+      fprintf(stderr,+              "Note: Device limits default grid size to %zu (down from %zu).\n",+              ctx->max_grid_size, cfg->default_grid_size);+    }+    cfg->default_grid_size = ctx->max_grid_size;+  }+  if (cfg->default_tile_size > ctx->max_tile_size) {+    if (cfg->default_tile_size_changed) {+      fprintf(stderr,+              "Note: Device limits default tile size to %zu (down from %zu).\n",+              ctx->max_tile_size, cfg->default_tile_size);+    }+    cfg->default_tile_size = ctx->max_tile_size;+  }++  if (!cfg->default_grid_size_changed) {+    cfg->default_grid_size =+      (device_query(ctx->dev, hipDeviceAttributePhysicalMultiProcessorCount) *+       device_query(ctx->dev, hipDeviceAttributeMaxThreadsPerMultiProcessor))+      / cfg->default_block_size;+  }++  for (int i = 0; i < cfg->num_tuning_params; i++) {+    const char *size_class = cfg->tuning_param_classes[i];+    int64_t *size_value = &cfg->tuning_params[i];+    const char* size_name = cfg->tuning_param_names[i];+    int64_t max_value = 0, default_value = 0;++    if (strstr(size_class, "group_size") == size_class) {+      max_value = ctx->max_group_size;+      default_value = cfg->default_block_size;+    } else if (strstr(size_class, "num_groups") == size_class) {+      max_value = ctx->max_grid_size;+      default_value = cfg->default_grid_size;+      // XXX: as a quick and dirty hack, use twice as many threads for+      // histograms by default.  We really should just be smarter+      // about sizes somehow.+      if (strstr(size_name, ".seghist_") != NULL) {+        default_value *= 2;+      }+    } else if (strstr(size_class, "tile_size") == size_class) {+      max_value = ctx->max_tile_size;+      default_value = cfg->default_tile_size;+    } else if (strstr(size_class, "reg_tile_size") == size_class) {+      max_value = 0; // No limit.+      default_value = cfg->default_reg_tile_size;+    } else if (strstr(size_class, "threshold") == size_class) {+      // Threshold can be as large as it takes.+      default_value = cfg->default_threshold;+    } else {+      // Bespoke sizes have no limit or default.+    }++    if (*size_value == 0) {+      *size_value = default_value;+    } else if (max_value > 0 && *size_value > max_value) {+      fprintf(stderr, "Note: Device limits %s to %zu (down from %zu)\n",+              size_name, max_value, *size_value);+      *size_value = max_value;+    }+  }+}++static char* hiprtc_build(const char *src, const char *opts[], size_t n_opts,+                          char **code, size_t *code_size) {+  hiprtcProgram prog;+  char *problem = NULL;++  problem = HIPRTC_SUCCEED_NONFATAL(hiprtcCreateProgram(&prog, src, "futhark-hip", 0, NULL, NULL));++  if (problem) {+    return problem;+  }++  hiprtcResult res = hiprtcCompileProgram(prog, n_opts, opts);+  if (res != HIPRTC_SUCCESS) {+    size_t log_size;+    if (hiprtcGetProgramLogSize(prog, &log_size) == HIPRTC_SUCCESS) {+      char *log = (char*) malloc(log_size+1);+      log[log_size] = 0; // HIPRTC does not zero-terminate.+      if (hiprtcGetProgramLog(prog, log) == HIPRTC_SUCCESS) {+        problem = msgprintf("HIPRTC compilation failed.\n\n%s\n", log);+      } else {+        problem = msgprintf("Could not retrieve compilation log\n");+      }+      free(log);+    }+    return problem;+  }++  HIPRTC_SUCCEED_FATAL(hiprtcGetCodeSize(prog, code_size));+  *code = (char*) malloc(*code_size);+  HIPRTC_SUCCEED_FATAL(hiprtcGetCode(prog, *code));+  HIPRTC_SUCCEED_FATAL(hiprtcDestroyProgram(&prog));+  return NULL;+}++static void hiprtc_mk_build_options(struct futhark_context *ctx, const char *extra_opts[],+                                    char*** opts_out, size_t *n_opts) {+  int arch_set = 0, num_extra_opts;+  struct futhark_context_config *cfg = ctx->cfg;++  for (num_extra_opts = 0; extra_opts[num_extra_opts] != NULL; num_extra_opts++) {+    if (strstr(extra_opts[num_extra_opts], "--gpu-architecture")+        == extra_opts[num_extra_opts]) {+      arch_set = 1;+    }+  }++  size_t i = 0, n_opts_alloc = 20 + num_extra_opts + cfg->num_tuning_params;+  char **opts = (char**) malloc(n_opts_alloc * sizeof(char *));+  if (!arch_set) {+    hipDeviceProp_t props;+    HIP_SUCCEED_FATAL(hipGetDeviceProperties(&props, ctx->dev_id));+    opts[i++] = msgprintf("--gpu-architecture=%s", props.gcnArchName);+  }+  if (cfg->debugging) {+    opts[i++] = strdup("-G");+    opts[i++] = strdup("-lineinfo");+  }+  opts[i++] = msgprintf("-D%s=%d",+                        "max_group_size",+                        (int)ctx->max_group_size);+  for (int j = 0; j < cfg->num_tuning_params; j++) {+    opts[i++] = msgprintf("-D%s=%zu", cfg->tuning_param_vars[j],+                          cfg->tuning_params[j]);+  }+  opts[i++] = msgprintf("-DLOCKSTEP_WIDTH=%zu", ctx->lockstep_width);+  opts[i++] = msgprintf("-DMAX_THREADS_PER_BLOCK=%zu", ctx->max_group_size);++  for (int j = 0; extra_opts[j] != NULL; j++) {+    opts[i++] = strdup(extra_opts[j]);+  }++  opts[i++] = msgprintf("-DTR_BLOCK_DIM=%d", TR_BLOCK_DIM);+  opts[i++] = msgprintf("-DTR_TILE_DIM=%d", TR_TILE_DIM);+  opts[i++] = msgprintf("-DTR_ELEMS_PER_THREAD=%d", TR_ELEMS_PER_THREAD);++  *n_opts = i;+  *opts_out = opts;+}++static char* hip_module_setup(struct futhark_context *ctx,+                              const char *src,+                              const char *extra_opts[],+                              const char* cache_fname) {+  char *code = NULL;+  size_t code_size = 0;+  struct futhark_context_config *cfg = ctx->cfg;++  char **opts;+  size_t n_opts;+  hiprtc_mk_build_options(ctx, extra_opts, &opts, &n_opts);++  if (cfg->logging) {+    fprintf(stderr, "HIPRTC build options:\n");+    for (size_t j = 0; j < n_opts; j++) {+      fprintf(stderr, "\t%s\n", opts[j]);+    }+    fprintf(stderr, "\n");+  }++  struct cache_hash h;+  int loaded_code_from_cache = 0;+  if (cache_fname != NULL) {+    hip_load_code_from_cache(cfg, src, (const char**)opts, n_opts, &h, cache_fname, &code, &code_size);++    if (code != NULL) {+      if (cfg->logging) {+        fprintf(stderr, "Restored compiled code from cache; now loading module...\n");+      }+      if (hipModuleLoadData(&ctx->module, code) == hipSuccess) {+        if (cfg->logging) {+          fprintf(stderr, "Success!\n");+        }+        loaded_code_from_cache = 1;+      } else {+        if (cfg->logging) {+          fprintf(stderr, "Failed!\n");+        }+        free(code);+        code = NULL;+      }+    }+  }++  if (code == NULL) {+    char* problem = hiprtc_build(src, (const char**)opts, n_opts, &code, &code_size);+    if (problem != NULL) {+      return problem;+    }+  }++  if (!loaded_code_from_cache) {+    HIP_SUCCEED_FATAL(hipModuleLoadData(&ctx->module, code));+  }++  if (cache_fname != NULL && !loaded_code_from_cache) {+    if (cfg->logging) {+      fprintf(stderr, "Caching compiled code in %s...\n", cache_fname);+    }+    errno = 0;+    if (cache_store(cache_fname, &h, (const unsigned char*)code, code_size) != 0) {+      fprintf(stderr, "Failed to cache compiled code: %s\n", strerror(errno));+    }+  }++  for (size_t i = 0; i < n_opts; i++) {+    free((char *)opts[i]);+  }+  free(opts);+  free(code);++  return NULL;+}++static int tally_profiling_records(struct futhark_context *ctx,+                                   struct cost_centres* ccs) {+  hipError_t err;+  for (int i = 0; i < ctx->profiling_records_used; i++) {+    struct profiling_record record = ctx->profiling_records[i];++    float ms;+    if ((err = hipEventElapsedTime(&ms, record.events[0], record.events[1])) != hipSuccess) {+      return err;+    }++    if (ccs) {+      struct cost_centre c = {+        .name = record.name,+        .runs = 1,+        .runtime = ms*1000+      };+      cost_centres_add(ccs, c);+    }++    if ((err = hipEventDestroy(record.events[0])) != hipSuccess) {+      return 1;+    }+    if ((err = hipEventDestroy(record.events[1])) != hipSuccess) {+      return 1;+    }++    free(record.events);+  }++  ctx->profiling_records_used = 0;++  return 0;+}++static hipEvent_t* hip_get_events(struct futhark_context *ctx, const char* name) {+  if (ctx->profiling_records_used == ctx->profiling_records_capacity) {+    ctx->profiling_records_capacity *= 2;+    ctx->profiling_records =+      realloc(ctx->profiling_records,+              ctx->profiling_records_capacity *+              sizeof(struct profiling_record));+  }+  hipEvent_t *events = calloc(2, sizeof(hipEvent_t));+  hipEventCreate(&events[0]);+  hipEventCreate(&events[1]);+  ctx->profiling_records[ctx->profiling_records_used].events = events;+  ctx->profiling_records[ctx->profiling_records_used].name = name;+  ctx->profiling_records_used++;+  return events;+}++int futhark_context_sync(struct futhark_context* ctx) {+  HIP_SUCCEED_OR_RETURN(hipStreamSynchronize(ctx->stream));+  if (ctx->failure_is_an_option) {+    // Check for any delayed error.+    int32_t failure_idx;+    HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(&failure_idx,+                                        ctx->global_failure,+                                        sizeof(int32_t)));+    ctx->failure_is_an_option = 0;++    if (failure_idx >= 0) {+      // We have to clear global_failure so that the next entry point+      // is not considered a failure from the start.+      int32_t no_failure = -1;+      HIP_SUCCEED_OR_RETURN(hipMemcpyHtoD(ctx->global_failure,+                                          &no_failure,+                                          sizeof(int32_t)));++      int64_t args[max_failure_args+1];+      HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(&args,+                                          ctx->global_failure_args,+                                          sizeof(args)));++      ctx->error = get_failure_msg(failure_idx, args);++      return FUTHARK_PROGRAM_ERROR;+    }+  }+  return 0;+}++struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx);+void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels);++int backend_context_setup(struct futhark_context* ctx) {+  ctx->profiling_records_capacity = 200;+  ctx->profiling_records_used = 0;+  ctx->profiling_records =+    malloc(ctx->profiling_records_capacity *+           sizeof(struct profiling_record));+  ctx->failure_is_an_option = 0;+  ctx->total_runs = 0;+  ctx->total_runtime = 0;+  ctx->peak_mem_usage_device = 0;+  ctx->cur_mem_usage_device = 0;++  HIP_SUCCEED_FATAL(hipInit(0));+  if (hip_device_setup(ctx) != 0) {+    futhark_panic(-1, "No suitable HIP device found.\n");+  }++  free_list_init(&ctx->gpu_free_list);++  ctx->max_local_memory = device_query(ctx->dev, hipDeviceAttributeMaxSharedMemoryPerBlock);+  ctx->max_group_size = device_query(ctx->dev, hipDeviceAttributeMaxThreadsPerBlock);+  ctx->max_grid_size = device_query(ctx->dev, hipDeviceAttributeMaxGridDimX);+  ctx->max_tile_size = sqrt(ctx->max_group_size);+  ctx->max_threshold = 0;+  ctx->max_bespoke = 0;+  // FIXME: in principle we should query hipDeviceAttributeWarpSize+  // from the device, which will provide 64 on AMD GPUs.+  // Unfortunately, we currently do nasty implicit intra-warp+  // synchronisation in codegen, which does not work when this is 64.+  // Once our codegen properly synchronises intra-warp operations, we+  // can use the actual hardware lockstep width instead.+  ctx->lockstep_width = 32;+  HIP_SUCCEED_FATAL(hipStreamCreate(&ctx->stream));+  hip_size_setup(ctx);+  ctx->error = hip_module_setup(ctx, ctx->cfg->program,+                                ctx->cfg->build_opts, ctx->cfg->cache_fname);++  if (ctx->error != NULL) {+    futhark_panic(1, "During HIP initialisation:\n%s\n", ctx->error);+  }++  int32_t no_error = -1;+  HIP_SUCCEED_FATAL(hipMalloc(&ctx->global_failure, sizeof(no_error)));+  HIP_SUCCEED_FATAL(hipMemcpyHtoD(ctx->global_failure, &no_error, sizeof(no_error)));+  // The +1 is to avoid zero-byte allocations.+  HIP_SUCCEED_FATAL(hipMalloc(&ctx->global_failure_args, sizeof(int64_t)*(max_failure_args+1)));++  if ((ctx->kernels = init_builtin_kernels(ctx)) == NULL) {+    return 1;+  }++  return 0;+}++void backend_context_teardown(struct futhark_context* ctx) {+  free_builtin_kernels(ctx, ctx->kernels);+  hipFree(ctx->global_failure);+  hipFree(ctx->global_failure_args);+  HIP_SUCCEED_FATAL(gpu_free_all(ctx));+  (void)tally_profiling_records(ctx, NULL);+  free(ctx->profiling_records);+  HIP_SUCCEED_FATAL(hipStreamDestroy(ctx->stream));+  HIP_SUCCEED_FATAL(hipModuleUnload(ctx->module));+}++// GPU ABSTRACTION LAYER++typedef hipFunction_t gpu_kernel;+typedef hipDeviceptr_t gpu_mem;++static void gpu_create_kernel(struct futhark_context *ctx,+                              gpu_kernel* kernel,+                              const char* name) {+  if (ctx->debugging) {+    fprintf(ctx->log, "Creating kernel %s.\n", name);+  }+  HIP_SUCCEED_FATAL(hipModuleGetFunction(kernel, ctx->module, name));+}++static void gpu_free_kernel(struct futhark_context *ctx,+                            gpu_kernel kernel) {+  (void)ctx;+  (void)kernel;+}++static int gpu_scalar_to_device(struct futhark_context* ctx,+                                gpu_mem dst, size_t offset, size_t size,+                                void *src) {+  hipEvent_t *pevents = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    pevents = hip_get_events(ctx, "copy_scalar_to_dev");+    HIP_SUCCEED_FATAL(hipEventRecord(pevents[0], ctx->stream));+  }+  HIP_SUCCEED_OR_RETURN(hipMemcpyHtoD((unsigned char*)dst + offset, src, size));+  if (pevents != NULL) {+    HIP_SUCCEED_FATAL(hipEventRecord(pevents[1], ctx->stream));+  }+  return FUTHARK_SUCCESS;+}++static int gpu_scalar_from_device(struct futhark_context* ctx,+                                  void *dst,+                                  gpu_mem src, size_t offset, size_t size) {+  hipEvent_t *pevents = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    pevents = hip_get_events(ctx, "copy_scalar_from_dev");+    HIP_SUCCEED_FATAL(hipEventRecord(pevents[0], ctx->stream));+  }+  HIP_SUCCEED_OR_RETURN(hipMemcpyDtoH(dst, (unsigned char*)src + offset, size));+  if (pevents != NULL) {+    HIP_SUCCEED_FATAL(hipEventRecord(pevents[1], ctx->stream));+  }+  return FUTHARK_SUCCESS;+}++static int gpu_memcpy(struct futhark_context* ctx,+                      gpu_mem dst, int64_t dst_offset,+                      gpu_mem src, int64_t src_offset,+                      int64_t nbytes) {+  hipEvent_t *pevents = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    pevents = hip_get_events(ctx, "copy_dev_to_dev");+    HIP_SUCCEED_FATAL(hipEventRecord(pevents[0], ctx->stream));+  }+  HIP_SUCCEED_OR_RETURN(hipMemcpyWithStream((unsigned char*)dst+dst_offset, (unsigned char*)src+src_offset,+                                            nbytes, hipMemcpyDeviceToDevice ,ctx->stream));+  if (pevents != NULL) {+    HIP_SUCCEED_FATAL(hipEventRecord(pevents[1], ctx->stream));+  }+  return FUTHARK_SUCCESS;+}++static int memcpy_host2gpu(struct futhark_context* ctx, bool sync,+                           gpu_mem dst, int64_t dst_offset,+                           const unsigned char* src, int64_t src_offset,+                           int64_t nbytes) {+  if (nbytes > 0) {+    hipEvent_t* pevents = NULL;+    if (ctx->profiling && !ctx->profiling_paused) {+      pevents = hip_get_events(ctx, "copy_host_to_dev");+      HIP_SUCCEED_FATAL(hipEventRecord(pevents[0], ctx->stream));+    }+    if (sync) {+      HIP_SUCCEED_OR_RETURN+        (hipMemcpyHtoD((unsigned char*)dst + dst_offset,+                       (unsigned char*)src + src_offset, nbytes));+    } else {+      HIP_SUCCEED_OR_RETURN+        (hipMemcpyHtoDAsync((unsigned char*)dst + dst_offset,+                            (unsigned char*)src + src_offset,+                            nbytes, ctx->stream));+    }+    if (pevents != NULL) {+      HIP_SUCCEED_FATAL(hipEventRecord(pevents[1], ctx->stream));+    }+  }+  return FUTHARK_SUCCESS;+}++static int memcpy_gpu2host(struct futhark_context* ctx, bool sync,+                           unsigned char* dst, int64_t dst_offset,+                           gpu_mem src, int64_t src_offset,+                           int64_t nbytes) {+  if (nbytes > 0) {+    hipEvent_t* pevents = NULL;+    if (ctx->profiling && !ctx->profiling_paused) {+      pevents = hip_get_events(ctx, "copy_dev_to_host");+      HIP_SUCCEED_FATAL(hipEventRecord(pevents[0], ctx->stream));+    }+    if (sync) {+      HIP_SUCCEED_OR_RETURN+        (hipMemcpyDtoH(dst + dst_offset,+                       (unsigned char*)src + src_offset,+                       nbytes));+    } else {+      HIP_SUCCEED_OR_RETURN+        (hipMemcpyDtoHAsync(dst + dst_offset,+                            (unsigned char*)src + src_offset,+                            nbytes, ctx->stream));+    }+    if (sync &&+        ctx->failure_is_an_option &&+        futhark_context_sync(ctx) != 0) {+      return 1;+    }+  }+  return FUTHARK_SUCCESS;+}++static int gpu_launch_kernel(struct futhark_context* ctx,+                             gpu_kernel kernel, const char *name,+                             const int32_t grid[3],+                             const int32_t block[3],+                             unsigned int local_mem_bytes,+                             int num_args,+                             void* args[num_args],+                             size_t args_sizes[num_args]) {+  (void) args_sizes;+  int64_t time_start = 0, time_end = 0;+  if (ctx->logging) {+    fprintf(ctx->log,+            "Launching kernel %s with\n"+            "  grid=(%d,%d,%d)\n"+            "  block=(%d,%d,%d)\n"+            "  local memory=%d\n",+            name,+            grid[0], grid[1], grid[2],+            block[0], block[1], block[2],+            local_mem_bytes);+    time_start = get_wall_time();+  }++  hipEvent_t *pevents = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    pevents = hip_get_events(ctx, name);+    HIP_SUCCEED_FATAL(hipEventRecord(pevents[0], ctx->stream));+  }++  HIP_SUCCEED_OR_RETURN+    (hipModuleLaunchKernel(kernel,+                           grid[0], grid[1], grid[2],+                           block[0], block[1], block[2],+                           local_mem_bytes, ctx->stream,+                           args, NULL));++  if (pevents != NULL) {+    HIP_SUCCEED_FATAL(hipEventRecord(pevents[1], ctx->stream));+  }++  if (ctx->debugging) {+    HIP_SUCCEED_FATAL(hipStreamSynchronize(ctx->stream));+    time_end = get_wall_time();+    long int time_diff = time_end - time_start;+    fprintf(ctx->log, "  runtime: %ldus\n\n", time_diff);+  }++  return FUTHARK_SUCCESS;+}++static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out) {+  hipError_t res = hipMalloc(mem_out, size);+  if (res == hipErrorOutOfMemory) {+    return FUTHARK_OUT_OF_MEMORY;+  }+  HIP_SUCCEED_OR_RETURN(res);+  return FUTHARK_SUCCESS;+}++static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem) {+  (void)ctx;+  HIP_SUCCEED_OR_RETURN(hipFree(mem));+  return FUTHARK_SUCCESS;+}++// End of backends/hip.h.

@@ -1,5 +1,18 @@ // Start of backends/opencl.h +// Note [32-bit transpositions]+//+// Transposition kernels are much slower when they have to use 64-bit+// arithmetic.  I observed about 0.67x slowdown on an A100 GPU when+// transposing four-byte elements (much less when transposing 8-byte+// elements).  Unfortunately, 64-bit arithmetic is a requirement for+// large arrays (see #1953 for what happens otherwise).  We generate+// both 32- and 64-bit index arithmetic versions of transpositions,+// and dynamically pick between them at runtime.  This is an+// unfortunate code bloat, and it would be preferable if we could+// simply optimise the 64-bit version to make this distinction+// unnecessary.  Fortunately these kernels are quite small.+ // Forward declarations. struct opencl_device_option; // Invoked by setup_opencl() after the platform and device has been@@ -117,13 +130,12 @@   const char** tuning_param_classes;   // Uniform fields above. +  char* program;   int preferred_device_num;   const char *preferred_platform;   const char *preferred_device;   int ignore_blacklist; -  const char* dump_program_to;-  const char* load_program_from;   const char* dump_binary_to;   const char* load_binary_from; @@ -150,10 +162,9 @@   cfg->preferred_platform = "";   cfg->preferred_device = "";   cfg->ignore_blacklist = 0;-  cfg->dump_program_to = NULL;-  cfg->load_program_from = NULL;   cfg->dump_binary_to = NULL;   cfg->load_binary_from = NULL;+  cfg->program = strconcat(gpu_program);    // The following are dummy sizes that mean the concrete defaults   // will be set during initialisation via hardware-inspection-based@@ -172,6 +183,7 @@  static void backend_context_config_teardown(struct futhark_context_config* cfg) {   free(cfg->build_opts);+  free(cfg->program); }  void futhark_context_config_add_build_option(struct futhark_context_config* cfg, const char *opt) {@@ -392,12 +404,12 @@   free(devices); } -void futhark_context_config_dump_program_to(struct futhark_context_config *cfg, const char *path) {-  cfg->dump_program_to = path;+const char* futhark_context_config_get_program(struct futhark_context_config *cfg) {+  return cfg->program; } -void futhark_context_config_load_program_from(struct futhark_context_config *cfg, const char *path) {-  cfg->load_program_from = path;+void futhark_context_config_set_program(struct futhark_context_config *cfg, const char *s) {+  cfg->program = strdup(s); }  void futhark_context_config_dump_binary_to(struct futhark_context_config *cfg, const char *path) {@@ -465,8 +477,7 @@ // A record of something that happened. struct profiling_record {   cl_event *event;-  int *runs;-  int64_t *runtime;+  const char* name; };  struct futhark_context {@@ -503,7 +514,7 @@   cl_command_queue queue;   cl_program clprogram; -  struct free_list cl_free_list;+  struct free_list gpu_free_list;    size_t max_group_size;   size_t max_num_groups;@@ -517,9 +528,10 @@   int profiling_records_capacity;   int profiling_records_used; +  struct builtin_kernels* kernels; }; -static cl_build_status build_opencl_program(cl_program program, cl_device_id device, const char* options) {+static cl_build_status build_gpu_program(cl_program program, cl_device_id device, const char* options) {   cl_int clBuildProgram_error = clBuildProgram(program, 1, &device, options, NULL, NULL);    // Avoid termination due to CL_BUILD_PROGRAM_FAILURE@@ -591,6 +603,10 @@                   "%s ", extra_build_opts[i]);   } +  w += snprintf(compile_opts+w, compile_opts_size-w,+                "-DTR_BLOCK_DIM=%d -DTR_TILE_DIM=%d -DTR_ELEMS_PER_THREAD=%d ",+                TR_BLOCK_DIM, TR_TILE_DIM, TR_ELEMS_PER_THREAD);+   // Oclgrind claims to support cl_khr_fp16, but this is not actually   // the case.   if (strcmp(device_option.platform_name, "Oclgrind") == 0) {@@ -600,49 +616,49 @@   return compile_opts; } + // Count up the runtime all the profiling_records that occured during execution. // Also clears the buffer of profiling_records.-static cl_int opencl_tally_profiling_records(struct futhark_context *ctx) {+static void tally_profiling_records(struct futhark_context *ctx,+                                    struct cost_centres* ccs) {   cl_int err;   for (int i = 0; i < ctx->profiling_records_used; i++) {     struct profiling_record record = ctx->profiling_records[i];      cl_ulong start_t, end_t; -    if ((err = clGetEventProfilingInfo(*record.event,-                                       CL_PROFILING_COMMAND_START,-                                       sizeof(start_t),-                                       &start_t,-                                       NULL)) != CL_SUCCESS) {-      return err;-    }--    if ((err = clGetEventProfilingInfo(*record.event,-                                       CL_PROFILING_COMMAND_END,-                                       sizeof(end_t),-                                       &end_t,-                                       NULL)) != CL_SUCCESS) {-      return err;-    }+    OPENCL_SUCCEED_FATAL(clGetEventProfilingInfo(*record.event,+                                                 CL_PROFILING_COMMAND_START,+                                                 sizeof(start_t),+                                                 &start_t,+                                                 NULL)); -    // OpenCL provides nanosecond resolution, but we want-    // microseconds.-    *record.runs += 1;-    *record.runtime += (end_t - start_t)/1000;+    OPENCL_SUCCEED_FATAL(clGetEventProfilingInfo(*record.event,+                                                 CL_PROFILING_COMMAND_END,+                                                 sizeof(end_t),+                                                 &end_t,+                                                 NULL)); -    if ((err = clReleaseEvent(*record.event)) != CL_SUCCESS) {-      return err;+    if (ccs) {+      // Note that OpenCL provides nanosecond resolution, but we want+      // microseconds.+      struct cost_centre c = {+        .name = record.name,+        .runs = 1,+        .runtime = (end_t - start_t)/1000+      };+      cost_centres_add(ccs, c);     }++    OPENCL_SUCCEED_FATAL(clReleaseEvent(*record.event));     free(record.event);   }    ctx->profiling_records_used = 0;--  return CL_SUCCESS; }  // If profiling, produce an event associated with a profiling record.-static cl_event* opencl_get_event(struct futhark_context *ctx, int *runs, int64_t *runtime) {+static cl_event* opencl_get_event(struct futhark_context *ctx, const char *name) {   if (ctx->profiling_records_used == ctx->profiling_records_capacity) {     ctx->profiling_records_capacity *= 2;     ctx->profiling_records =@@ -652,130 +668,11 @@   }   cl_event *event = malloc(sizeof(cl_event));   ctx->profiling_records[ctx->profiling_records_used].event = event;-  ctx->profiling_records[ctx->profiling_records_used].runs = runs;-  ctx->profiling_records[ctx->profiling_records_used].runtime = runtime;+  ctx->profiling_records[ctx->profiling_records_used].name = name;   ctx->profiling_records_used++;   return event; } -// Allocate memory from driver. The problem is that OpenCL may perform-// lazy allocation, so we cannot know whether an allocation succeeded-// until the first time we try to use it.  Hence we immediately-// perform a write to see if the allocation succeeded.  This is slow,-// but the assumption is that this operation will be rare (most things-// will go through the free list).-static int opencl_alloc_actual(struct futhark_context *ctx, size_t size, cl_mem *mem_out) {-  int error;-  *mem_out = clCreateBuffer(ctx->ctx, CL_MEM_READ_WRITE, size, NULL, &error);--  if (error != CL_SUCCESS) {-    return error;-  }--  int x = 2;-  error = clEnqueueWriteBuffer(ctx->queue, *mem_out,-                               CL_TRUE,-                               0, sizeof(x), &x,-                               0, NULL, NULL);--  // No need to wait for completion here. clWaitForEvents() cannot-  // return mem object allocation failures. This implies that the-  // buffer is faulted onto the device on enqueue. (Observation by-  // Andreas Kloeckner.)--  return error;-}--static int opencl_alloc(struct futhark_context *ctx, FILE *log,-                        size_t min_size, const char *tag,-                        cl_mem *mem_out, size_t *size_out) {-  (void)tag;-  if (min_size < sizeof(int)) {-    min_size = sizeof(int);-  }--  cl_mem* memptr;-  if (free_list_find(&ctx->cl_free_list, min_size, tag, size_out, (fl_mem*)&memptr) == 0) {-    // Successfully found a free block.  Is it big enough?-    if (*size_out >= min_size) {-      if (ctx->cfg->debugging) {-        fprintf(log, "No need to allocate: Found a block in the free list.\n");-      }-      *mem_out = *memptr;-      free(memptr);-      return CL_SUCCESS;-    } else {-      if (ctx->cfg->debugging) {-        fprintf(log, "Found a free block, but it was too small.\n");-      }-      int error = clReleaseMemObject(*memptr);-      free(*memptr);-      if (error != CL_SUCCESS) {-        return error;-      }-    }-  }--  *size_out = min_size;--  // We have to allocate a new block from the driver.  If the-  // allocation does not succeed, then we might be in an out-of-memory-  // situation.  We now start freeing things from the free list until-  // we think we have freed enough that the allocation will succeed.-  // Since we don't know how far the allocation is from fitting, we-  // have to check after every deallocation.  This might be pretty-  // expensive.  Let's hope that this case is hit rarely.--  if (ctx->cfg->debugging) {-    fprintf(log, "Actually allocating the desired block.\n");-  }--  int error = opencl_alloc_actual(ctx, min_size, mem_out);--  while (error == CL_MEM_OBJECT_ALLOCATION_FAILURE) {-    if (ctx->cfg->debugging) {-      fprintf(log, "Out of OpenCL memory: releasing entry from the free list...\n");-    }-    cl_mem* memptr;-    if (free_list_first(&ctx->cl_free_list, (fl_mem*)&memptr) == 0) {-      cl_mem mem = *memptr;-      free(memptr);-      error = clReleaseMemObject(mem);-      if (error != CL_SUCCESS) {-        return error;-      }-    } else {-      break;-    }-    error = opencl_alloc_actual(ctx, min_size, mem_out);-  }--  return error;-}--static int opencl_free(struct futhark_context *ctx,-                       cl_mem mem, size_t size, const char *tag) {-  cl_mem* memptr = malloc(sizeof(cl_mem));-  *memptr = mem;-  free_list_insert(&ctx->cl_free_list, size, (fl_mem)memptr, tag);-  return CL_SUCCESS;-}--static int opencl_free_all(struct futhark_context *ctx) {-  free_list_pack(&ctx->cl_free_list);-  cl_mem* memptr;-  while (free_list_first(&ctx->cl_free_list, (fl_mem*)&memptr) == 0) {-    cl_mem mem = *memptr;-    free(memptr);-    int error = clReleaseMemObject(mem);-    if (error != CL_SUCCESS) {-      return error;-    }-  }--  return CL_SUCCESS;-}- int futhark_context_sync(struct futhark_context* ctx) {   // Check for any delayed error.   cl_int failure_idx = -1;@@ -822,12 +719,11 @@ // array must be NULL-terminated. static void setup_opencl_with_command_queue(struct futhark_context *ctx,                                             cl_command_queue queue,-                                            const char *srcs[],                                             const char *extra_build_opts[],                                             const char* cache_fname) {   int error; -  free_list_init(&ctx->cl_free_list);+  free_list_init(&ctx->gpu_free_list);   ctx->queue = queue;    OPENCL_SUCCEED_FATAL(clGetCommandQueueInfo(ctx->queue, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx->ctx, NULL));@@ -971,7 +867,7 @@     fprintf(stderr, "OpenCL compiler options: %s\n", compile_opts);   } -  char *fut_opencl_src = NULL;+  const char* opencl_src = ctx->cfg->program;   cl_program prog;   error = CL_SUCCESS; @@ -981,40 +877,12 @@   if (ctx->cfg->load_binary_from == NULL) {     size_t src_size = 0; -    // Maybe we have to read OpenCL source from somewhere else (used for debugging).-    if (ctx->cfg->load_program_from != NULL) {-      fut_opencl_src = slurp_file(ctx->cfg->load_program_from, NULL);-      assert(fut_opencl_src != NULL);-    } else {-      // Construct the OpenCL source concatenating all the fragments.-      for (const char **src = srcs; src && *src; src++) {-        src_size += strlen(*src);-      }--      fut_opencl_src = (char*) malloc(src_size + 1);--      size_t n, i;-      for (i = 0, n = 0; srcs && srcs[i]; i++) {-        strncpy(fut_opencl_src+n, srcs[i], src_size-n);-        n += strlen(srcs[i]);-      }-      fut_opencl_src[src_size] = 0;-    }--    if (ctx->cfg->dump_program_to != NULL) {-      if (ctx->cfg->logging) {-        fprintf(stderr, "Dumping OpenCL source to %s...\n", ctx->cfg->dump_program_to);-      }--      dump_file(ctx->cfg->dump_program_to, fut_opencl_src, strlen(fut_opencl_src));-    }-     if (cache_fname != NULL) {       if (ctx->cfg->logging) {         fprintf(stderr, "Restoring cache from from %s...\n", cache_fname);       }       cache_hash_init(&h);-      cache_hash(&h, fut_opencl_src, strlen(fut_opencl_src));+      cache_hash(&h, opencl_src, strlen(opencl_src));       cache_hash(&h, compile_opts, strlen(compile_opts));        unsigned char *buf;@@ -1051,7 +919,7 @@         fprintf(stderr, "Creating OpenCL program...\n");       } -      const char* src_ptr[] = {fut_opencl_src};+      const char* src_ptr[] = {opencl_src};       prog = clCreateProgramWithSource(ctx->ctx, 1, src_ptr, &src_size, &error);       OPENCL_SUCCEED_FATAL(error);     }@@ -1077,10 +945,9 @@   if (ctx->cfg->logging) {     fprintf(stderr, "Building OpenCL program...\n");   }-  OPENCL_SUCCEED_FATAL(build_opencl_program(prog, device_option.device, compile_opts));+  OPENCL_SUCCEED_FATAL(build_gpu_program(prog, device_option.device, compile_opts));    free(compile_opts);-  free(fut_opencl_src);    size_t binary_size = 0;   unsigned char *binary = NULL;@@ -1144,7 +1011,6 @@ }  static void setup_opencl(struct futhark_context *ctx,-                         const char *srcs[],                          const char *extra_build_opts[],                          const char* cache_fname) {   struct opencl_device_option device_option = get_preferred_device(ctx->cfg);@@ -1173,9 +1039,12 @@                          &clCreateCommandQueue_error);   OPENCL_SUCCEED_FATAL(clCreateCommandQueue_error); -  setup_opencl_with_command_queue(ctx, queue, srcs, extra_build_opts, cache_fname);+  setup_opencl_with_command_queue(ctx, queue, extra_build_opts, cache_fname); } +struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx);+void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels);+ int backend_context_setup(struct futhark_context* ctx) {   ctx->lockstep_width = 0; // Real value set later.   ctx->profiling_records_capacity = 200;@@ -1190,9 +1059,9 @@   ctx->cur_mem_usage_device = 0;    if (ctx->cfg->queue_set) {-    setup_opencl_with_command_queue(ctx, ctx->cfg->queue, opencl_program, ctx->cfg->build_opts, ctx->cfg->cache_fname);+    setup_opencl_with_command_queue(ctx, ctx->cfg->queue, ctx->cfg->build_opts, ctx->cfg->cache_fname);   } else {-    setup_opencl(ctx, opencl_program, ctx->cfg->build_opts, ctx->cfg->cache_fname);+    setup_opencl(ctx, ctx->cfg->build_opts, ctx->cfg->cache_fname);   }    cl_int error;@@ -1209,15 +1078,23 @@                    CL_MEM_READ_WRITE,                    sizeof(int64_t)*(max_failure_args+1), NULL, &error);   OPENCL_SUCCEED_OR_RETURN(error);-  return 0;++  if ((ctx->kernels = init_builtin_kernels(ctx)) == NULL) {+    return 1;+  }++  return FUTHARK_SUCCESS; } +static int gpu_free_all(struct futhark_context *ctx);+ void backend_context_teardown(struct futhark_context* ctx) {+  free_builtin_kernels(ctx, ctx->kernels);   OPENCL_SUCCEED_FATAL(clReleaseMemObject(ctx->global_failure));   OPENCL_SUCCEED_FATAL(clReleaseMemObject(ctx->global_failure_args));-  (void)opencl_tally_profiling_records(ctx);+  (void)tally_profiling_records(ctx, NULL);   free(ctx->profiling_records);-  (void)opencl_free_all(ctx);+  (void)gpu_free_all(ctx);   (void)clReleaseProgram(ctx->clprogram);   (void)clReleaseCommandQueue(ctx->queue);   (void)clReleaseContext(ctx->ctx);@@ -1225,6 +1102,231 @@  cl_command_queue futhark_context_get_command_queue(struct futhark_context* ctx) {   return ctx->queue;+}++// GPU ABSTRACTION LAYER++// Types.++typedef cl_kernel gpu_kernel;+typedef cl_mem gpu_mem;++static void gpu_create_kernel(struct futhark_context *ctx,+                              gpu_kernel* kernel,+                              const char* name) {+  if (ctx->debugging) {+    fprintf(ctx->log, "Creating kernel %s.\n", name);+  }+  cl_int error;+  *kernel = clCreateKernel(ctx->clprogram, name, &error);+  OPENCL_SUCCEED_FATAL(error);+}++static void gpu_free_kernel(struct futhark_context *ctx,+                            gpu_kernel kernel) {+  (void)ctx;+  clReleaseKernel(kernel);+}++static int gpu_scalar_to_device(struct futhark_context* ctx,+                                gpu_mem dst, size_t offset, size_t size,+                                void *src) {+  cl_event* event = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    event = opencl_get_event(ctx, "copy_scalar_to_dev");+  }+  OPENCL_SUCCEED_OR_RETURN+    (clEnqueueWriteBuffer+     (ctx->queue, dst, CL_TRUE,+      offset, size, src, 0, NULL, event));+  return 0;+}++static int gpu_scalar_from_device(struct futhark_context* ctx,+                                  void *dst,+                                  gpu_mem src, size_t offset, size_t size) {+  cl_event* event = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    event = opencl_get_event(ctx, "copy_scalar_from_dev");+  }+  OPENCL_SUCCEED_OR_RETURN+    (clEnqueueReadBuffer+     (ctx->queue, src, ctx->failure_is_an_option ? CL_FALSE : CL_TRUE,+      offset, size, dst, 0, NULL, event));+  return 0;+}++static int gpu_memcpy(struct futhark_context* ctx,+                      gpu_mem dst, int64_t dst_offset,+                      gpu_mem src, int64_t src_offset,+                      int64_t nbytes) {+  if (nbytes > 0) {+    cl_event* event = NULL;+    if (ctx->profiling && !ctx->profiling_paused) {+      event = opencl_get_event(ctx, "copy_dev_to_dev");+    }+    // OpenCL swaps the usual order of operands for memcpy()-like+    // functions.  The order below is not a typo.+    OPENCL_SUCCEED_OR_RETURN+      (clEnqueueCopyBuffer+       (ctx->queue, src, dst, src_offset, dst_offset, nbytes,+        0, NULL, event));+    if (ctx->debugging) {+      OPENCL_SUCCEED_FATAL(clFinish(ctx->queue));+    }+  }+  return FUTHARK_SUCCESS;+}++static int memcpy_host2gpu(struct futhark_context* ctx, bool sync,+                           gpu_mem dst, int64_t dst_offset,+                           const unsigned char* src, int64_t src_offset,+                           int64_t nbytes) {+  if (nbytes > 0) {+    cl_event* event = NULL;+    if (ctx->profiling && !ctx->profiling_paused) {+      event = opencl_get_event(ctx, "copy_host_to_dev");+    }+    OPENCL_SUCCEED_OR_RETURN+      (clEnqueueWriteBuffer(ctx->queue,+                            dst,+                            sync ? CL_TRUE : CL_FALSE,+                            (size_t)dst_offset, (size_t)nbytes,+                            src + src_offset,+                            0, NULL, event));+    if (ctx->debugging) {+      OPENCL_SUCCEED_FATAL(clFinish(ctx->queue));+    }+  }+  return FUTHARK_SUCCESS;+}++static int memcpy_gpu2host(struct futhark_context* ctx, bool sync,+                           unsigned char* dst, int64_t dst_offset,+                           gpu_mem src, int64_t src_offset,+                           int64_t nbytes) {+  if (nbytes > 0) {+    cl_event* event = NULL;+    if (ctx->profiling && !ctx->profiling_paused) {+      event = opencl_get_event(ctx, "copy_dev_to_host");+    }+    OPENCL_SUCCEED_OR_RETURN+      (clEnqueueReadBuffer(ctx->queue, src,+                           ctx->failure_is_an_option ? CL_FALSE+                           : sync ? CL_TRUE : CL_FALSE,+                           src_offset, nbytes,+                           dst + dst_offset,+                           0, NULL, event));+    if (sync &&+        ctx->failure_is_an_option &&+        futhark_context_sync(ctx) != 0) {+      return 1;+    }+  }+  return FUTHARK_SUCCESS;+}++static int gpu_launch_kernel(struct futhark_context* ctx,+                             gpu_kernel kernel, const char *name,+                             const int32_t grid[3],+                             const int32_t block[3],+                             unsigned int local_mem_bytes,+                             int num_args,+                             void* args[num_args],+                             size_t args_sizes[num_args]) {+  int64_t time_start = 0, time_end = 0;+  if (ctx->logging) {+    fprintf(ctx->log,+            "Launching kernel %s with\n"+            "  grid=(%d,%d,%d)\n"+            "  block=(%d,%d,%d)\n"+            "  local memory=%d\n",+            name,+            grid[0], grid[1], grid[2],+            block[0], block[1], block[2],+            local_mem_bytes);+    time_start = get_wall_time();+  }++  // Some implementations do not work with 0-byte local memory.+  if (local_mem_bytes == 0) {+    local_mem_bytes = 4;+  }++  OPENCL_SUCCEED_OR_RETURN+    (clSetKernelArg(kernel, 0, local_mem_bytes, NULL));+  for (int i = 0; i < num_args; i++) {+    OPENCL_SUCCEED_OR_RETURN+      (clSetKernelArg(kernel, i+1, args_sizes[i], args[i]));+  }++  const size_t global_work_size[3] =+    {(size_t)grid[0]*block[0],+     (size_t)grid[1]*block[1],+     (size_t)grid[2]*block[2]};+  const size_t local_work_size[3] =+    {block[0],+     block[1],+     block[2]};++  cl_event* event = NULL;+  if (ctx->profiling && !ctx->profiling_paused) {+    event = opencl_get_event(ctx, name);+  }++  OPENCL_SUCCEED_OR_RETURN+    (clEnqueueNDRangeKernel(ctx->queue,+                            kernel,+                            3, NULL, global_work_size, local_work_size,+                            0, NULL, event));++  if (ctx->debugging) {+    OPENCL_SUCCEED_FATAL(clFinish(ctx->queue));+    time_end = get_wall_time();+    long int time_diff = time_end - time_start;+    fprintf(ctx->log, "  runtime: %ldus\n", time_diff);+  }+  if (ctx->logging) {+    printf("\n");+  }++  return FUTHARK_SUCCESS;+}++// Allocate memory from driver. The problem is that OpenCL may perform+// lazy allocation, so we cannot know whether an allocation succeeded+// until the first time we try to use it.  Hence we immediately+// perform a write to see if the allocation succeeded.  This is slow,+// but the assumption is that this operation will be rare (most things+// will go through the free list).+static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out) {+  int error;+  *mem_out = clCreateBuffer(ctx->ctx, CL_MEM_READ_WRITE, size, NULL, &error);++  OPENCL_SUCCEED_OR_RETURN(error);++  int x = 2;+  error = clEnqueueWriteBuffer(ctx->queue, *mem_out,+                               CL_TRUE,+                               0, sizeof(x), &x,+                               0, NULL, NULL);++  // No need to wait for completion here. clWaitForEvents() cannot+  // return mem object allocation failures. This implies that the+  // buffer is faulted onto the device on enqueue. (Observation by+  // Andreas Kloeckner.)++  if (error == CL_MEM_OBJECT_ALLOCATION_FAILURE) {+    return FUTHARK_OUT_OF_MEMORY;+  }+  OPENCL_SUCCEED_OR_RETURN(error);+  return FUTHARK_SUCCESS;+}++static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem) {+  (void)ctx;+  OPENCL_SUCCEED_OR_RETURN(clReleaseMemObject(mem));+  return FUTHARK_SUCCESS; }  // End of backends/opencl.h

@@ -20,6 +20,27 @@ // Allocate memory allocated with host_alloc(). static void host_free(struct futhark_context* ctx, size_t size, const char* tag, void* mem); +// Log that a copy has occurred.+static void log_copy(struct futhark_context* ctx,+                     const char *kind, int r,+                     int64_t dst_offset, int64_t dst_strides[r],+                     int64_t src_offset, int64_t src_strides[r],+                     int64_t shape[r]);++static void log_transpose(struct futhark_context* ctx,+                          int64_t k, int64_t m, int64_t n);++static bool lmad_map_tr(int64_t *num_arrays_out, int64_t *n_out, int64_t *m_out,+                        int r,+                        const int64_t dst_strides[r],+                        const int64_t src_strides[r],+                        const int64_t shape[r]);++static bool lmad_contiguous(int r, int64_t strides[r], int64_t shape[r]);++static bool lmad_memcpyable(int r,+                            int64_t dst_strides[r], int64_t src_strides[r], int64_t shape[r]);+ // Functions that must be defined by the backend. static void backend_context_config_setup(struct futhark_context_config* cfg); static void backend_context_config_teardown(struct futhark_context_config* cfg);

@@ -0,0 +1,262 @@+// Start of copy.h++// Cache-oblivious map-transpose function.+#define GEN_MAP_TRANSPOSE(NAME, ELEM_TYPE)                              \+  static void map_transpose_##NAME                                      \+  (ELEM_TYPE* dst, ELEM_TYPE* src,                                      \+   int64_t k, int64_t m, int64_t n,                                     \+   int64_t cb, int64_t ce, int64_t rb, int64_t re)                      \+  {                                                                     \+  int32_t r = re - rb;                                                  \+  int32_t c = ce - cb;                                                  \+  if (k == 1) {                                                         \+    if (r <= 64 && c <= 64) {                                           \+      for (int64_t j = 0; j < c; j++) {                                 \+        for (int64_t i = 0; i < r; i++) {                               \+          dst[(j + cb) * n + (i + rb)] = src[(i + rb) * m + (j + cb)];  \+        }                                                               \+      }                                                                 \+    } else if (c <= r) {                                                \+      map_transpose_##NAME(dst, src, k, m, n, cb, ce, rb, rb + r/2);    \+      map_transpose_##NAME(dst, src, k, m, n, cb, ce, rb + r/2, re);    \+    } else {                                                            \+      map_transpose_##NAME(dst, src, k, m, n, cb, cb + c/2, rb, re);    \+      map_transpose_##NAME(dst, src, k, m, n, cb + c/2, ce, rb, re);    \+    }                                                                   \+  } else {                                                              \+  for (int64_t i = 0; i < k; i++) {                                     \+    map_transpose_##NAME(dst + i * m * n, src + i * m * n, 1, m, n, cb, ce, rb, re); \+  }\+} \+}++// Straightforward LMAD copy function.+#define GEN_LMAD_COPY_ELEMENTS(NAME, ELEM_TYPE)                         \+  static void lmad_copy_elements_##NAME(int r,                          \+                                        ELEM_TYPE* dst, int64_t dst_strides[r], \+                                        ELEM_TYPE *src, int64_t src_strides[r], \+                                        int64_t shape[r]) {             \+    if (r == 1) {                                                       \+      for (int i = 0; i < shape[0]; i++) {                              \+        dst[i*dst_strides[0]] = src[i*src_strides[0]];                  \+      }                                                                 \+    } else if (r > 1) {                                                 \+      for (int i = 0; i < shape[0]; i++) {                              \+        lmad_copy_elements_##NAME(r-1,                                  \+                                  dst+i*dst_strides[0], dst_strides+1,  \+                                  src+i*src_strides[0], src_strides+1,  \+                                  shape+1);                             \+      }                                                                 \+    }                                                                   \+  }                                                                     \++// Check whether this LMAD can be seen as a transposed 2D array.  This+// is done by checking every possible splitting point.+static bool lmad_is_tr(int64_t *n_out, int64_t *m_out,+                       int r,+                       const int64_t strides[r],+                       const int64_t shape[r]) {+  for (int i = 1; i < r; i++) {+    int n = 1, m = 1;+    bool ok = true;+    int64_t expected = 1;+    // Check strides before 'i'.+    for (int j = i-1; j >= 0; j--) {+      ok = ok && strides[j] == expected;+      expected *= shape[j];+      n *= shape[j];+    }+    // Check strides after 'i'.+    for (int j = r-1; j >= i; j--) {+      ok = ok && strides[j] == expected;+      expected *= shape[j];+      m *= shape[j];+    }+    if (ok) {+      *n_out = n;+      *m_out = m;+      return true;+    }+  }+  return false;+}++// This function determines whether the a 'dst' LMAD is row-major and+// 'src' LMAD is column-major.  Both LMADs are for arrays of the same+// shape.  Both LMADs are allowed to have additional dimensions "on+// top".  Essentially, this function determines whether a copy from+// 'src' to 'dst' is a "map(transpose)" that we know how to implement+// efficiently.  The LMADs can have arbitrary rank, and the main+// challenge here is checking whether the src LMAD actually+// corresponds to a 2D column-major layout by morally collapsing+// dimensions.  There is a lot of looping here, but the actual trip+// count is going to be very low in practice.+//+// Returns true if this is indeed a map(transpose), and writes the+// number of arrays, and moral array size to appropriate output+// parameters.+static bool lmad_map_tr(int64_t *num_arrays_out, int64_t *n_out, int64_t *m_out,+                        int r,+                        const int64_t dst_strides[r],+                        const int64_t src_strides[r],+                        const int64_t shape[r]) {+  int64_t rowmajor_strides[r];+  rowmajor_strides[r-1] = 1;++  for (int i = r-2; i >= 0; i--) {+    rowmajor_strides[i] = rowmajor_strides[i+1] * shape[i+1];+  }++  // map_r will be the number of mapped dimensions on top.+  int map_r = 0;+  int64_t num_arrays = 1;+  for (int i = 0; i < r; i++) {+    if (dst_strides[i] != rowmajor_strides[i] ||+        src_strides[i] != rowmajor_strides[i]) {+      break;+    } else {+      num_arrays *= shape[i];+      map_r++;+    }+  }++  *num_arrays_out = num_arrays;++  if (memcmp(&rowmajor_strides[map_r],+             &dst_strides[map_r],+             sizeof(int64_t)*(r-map_r)) == 0) {+    return lmad_is_tr(n_out, m_out, r-map_r, src_strides+map_r, shape+map_r);+  } else if (memcmp(&rowmajor_strides[map_r],+                    &src_strides[map_r],+                    sizeof(int64_t)*(r-map_r)) == 0) {+    return lmad_is_tr(m_out, n_out, r-map_r, dst_strides+map_r, shape+map_r);+  }+  return false;+}++// Check if the strides correspond to row-major strides of *any*+// permutation of the shape.  This is done by recursive search with+// backtracking.  This is worst-case exponential, but hopefully the+// arrays we encounter do not have that many dimensions.+static bool lmad_contiguous_search(int checked, int64_t expected,+                                   int r,+                                   int64_t strides[r], int64_t shape[r], bool used[r]) {+  for (int i = 0; i < r; i++) {+    for (int j = 0; j < r; j++) {+      if (!used[j] && strides[j] == expected && strides[j] >= 0) {+        used[j] = true;+        if (checked+1 == r ||+            lmad_contiguous_search(checked+1, expected * shape[j], r, strides, shape, used)) {+          return true;+        }+        used[j] = false;+      }+    }+  }+  return false;+}++// Does this LMAD correspond to an array with positive strides and no+// holes?+static bool lmad_contiguous(int r, int64_t strides[r], int64_t shape[r]) {+  bool used[r];+  for (int i = 0; i < r; i++) {+    used[i] = false;+  }+  return lmad_contiguous_search(0, 1, r, strides, shape, used);+}++// Does this copy correspond to something that could be done with a+// memcpy()-like operation?  I.e. do the LMADs actually represent the+// same in-memory layout and are they contiguous?+static bool lmad_memcpyable(int r,+                            int64_t dst_strides[r], int64_t src_strides[r], int64_t shape[r]) {+  if (!lmad_contiguous(r, dst_strides, shape)) {+    return false;+  }+  for (int i = 0; i < r; i++) {+    if (dst_strides[i] != src_strides[i] && shape[i] != 1) {+      return false;+    }+  }+  return true;+}+++static void log_copy(struct futhark_context* ctx,+                     const char *kind, int r,+                     int64_t dst_offset, int64_t dst_strides[r],+                     int64_t src_offset, int64_t src_strides[r],+                     int64_t shape[r]) {+  if (ctx->logging) {+    fprintf(ctx->log, "\n# Copy %s\n", kind);+    fprintf(ctx->log, "Shape: ");+    for (int i = 0; i < r; i++) { fprintf(ctx->log, "[%ld]", (long int)shape[i]); }+    fprintf(ctx->log, "\n");+    fprintf(ctx->log, "Dst offset: %ld\n", dst_offset);+    fprintf(ctx->log, "Dst strides:");+    for (int i = 0; i < r; i++) { fprintf(ctx->log, " %ld", (long int)dst_strides[i]); }+    fprintf(ctx->log, "\n");+    fprintf(ctx->log, "Src offset: %ld\n", src_offset);+    fprintf(ctx->log, "Src strides:");+    for (int i = 0; i < r; i++) { fprintf(ctx->log, " %ld", (long int)src_strides[i]); }+    fprintf(ctx->log, "\n");+  }+}++static void log_transpose(struct futhark_context* ctx,+                          int64_t k, int64_t n, int64_t m) {+  if (ctx->logging) {+    fprintf(ctx->log, "## Transpose\n");+    fprintf(ctx->log, "Arrays     : %ld\n", (long int)k);+    fprintf(ctx->log, "X elements : %ld\n", (long int)m);+    fprintf(ctx->log, "Y elements : %ld\n", (long int)n);+    fprintf(ctx->log, "\n");+  }+}++#define GEN_LMAD_COPY(NAME, ELEM_TYPE)                                  \+  static void lmad_copy_##NAME                                          \+  (struct futhark_context *ctx, int r,                                  \+   ELEM_TYPE* dst, int64_t dst_offset, int64_t dst_strides[r],          \+   ELEM_TYPE *src, int64_t src_offset, int64_t src_strides[r],          \+   int64_t shape[r]) {                                                  \+    log_copy(ctx, "CPU to CPU", r, dst_offset, dst_strides,             \+             src_offset, src_strides, shape);                           \+    int64_t size = 1;                                                   \+    for (int i = 0; i < r; i++) { size *= shape[i]; }                   \+    if (size == 0) { return; }                                          \+    int64_t k, n, m;                                                    \+    if (lmad_map_tr(&k, &n, &m,                                         \+                    r, dst_strides, src_strides, shape)) {              \+      log_transpose(ctx, k, n, m);                                      \+      map_transpose_##NAME                                              \+        (dst+dst_offset, src+src_offset, k, n, m, 0, n, 0, m);          \+    } else if (lmad_memcpyable(r, dst_strides, src_strides, shape)) {   \+      if (ctx->logging) {fprintf(ctx->log, "## Flat copy\n\n");}          \+      memcpy(dst+dst_offset, src+src_offset, size*sizeof(*dst));        \+    } else {                                                            \+      if (ctx->logging) {fprintf(ctx->log, "## General copy\n\n");}       \+      lmad_copy_elements_##NAME                                         \+        (r,                                                             \+         dst+dst_offset, dst_strides,                                   \+         src+src_offset, src_strides, shape);                           \+    }                                                                   \+  }++GEN_MAP_TRANSPOSE(1b, uint8_t)+GEN_MAP_TRANSPOSE(2b, uint16_t)+GEN_MAP_TRANSPOSE(4b, uint32_t)+GEN_MAP_TRANSPOSE(8b, uint64_t)++GEN_LMAD_COPY_ELEMENTS(1b, uint8_t)+GEN_LMAD_COPY_ELEMENTS(2b, uint16_t)+GEN_LMAD_COPY_ELEMENTS(4b, uint32_t)+GEN_LMAD_COPY_ELEMENTS(8b, uint64_t)++GEN_LMAD_COPY(1b, uint8_t)+GEN_LMAD_COPY(2b, uint16_t)+GEN_LMAD_COPY(4b, uint32_t)+GEN_LMAD_COPY(8b, uint64_t)++// End of copy.h

@@ -0,0 +1,591 @@+// Start of gpu.h++// Generic functions that use our tiny GPU abstraction layer.  The+// entire context must be defined before this header is included.  In+// particular we expect the following functions to be available:++static int gpu_free_actual(struct futhark_context *ctx, gpu_mem mem);+static int gpu_alloc_actual(struct futhark_context *ctx, size_t size, gpu_mem *mem_out);+int gpu_launch_kernel(struct futhark_context* ctx,+                      gpu_kernel kernel, const char *name,+                      const int32_t grid[3],+                      const int32_t block[3],+                      unsigned int local_mem_bytes,+                      int num_args,+                      void* args[num_args],+                      size_t args_sizes[num_args]);+int gpu_memcpy(struct futhark_context* ctx,+               gpu_mem dst, int64_t dst_offset,+               gpu_mem src, int64_t src_offset,+               int64_t nbytes);+int gpu_scalar_from_device(struct futhark_context* ctx,+                           void *dst,+                           gpu_mem src, size_t offset, size_t size);+int gpu_scalar_to_device(struct futhark_context* ctx,+                         gpu_mem dst, size_t offset, size_t size,+                         void *src);+void gpu_create_kernel(struct futhark_context *ctx,+                       gpu_kernel* kernel,+                       const char* name);++// Max number of groups we allow along the second or third dimension+// for transpositions.+#define MAX_TR_GROUPS 65535++struct builtin_kernels {+  // We have a lot of ways to transpose arrays.+  gpu_kernel map_transpose_1b;+  gpu_kernel map_transpose_1b_low_height;+  gpu_kernel map_transpose_1b_low_width;+  gpu_kernel map_transpose_1b_small;+  gpu_kernel map_transpose_1b_large;+  gpu_kernel map_transpose_2b;+  gpu_kernel map_transpose_2b_low_height;+  gpu_kernel map_transpose_2b_low_width;+  gpu_kernel map_transpose_2b_small;+  gpu_kernel map_transpose_2b_large;+  gpu_kernel map_transpose_4b;+  gpu_kernel map_transpose_4b_low_height;+  gpu_kernel map_transpose_4b_low_width;+  gpu_kernel map_transpose_4b_small;+  gpu_kernel map_transpose_4b_large;+  gpu_kernel map_transpose_8b;+  gpu_kernel map_transpose_8b_low_height;+  gpu_kernel map_transpose_8b_low_width;+  gpu_kernel map_transpose_8b_small;+  gpu_kernel map_transpose_8b_large;++  // And a few ways of copying.+  gpu_kernel lmad_copy_1b;+  gpu_kernel lmad_copy_2b;+  gpu_kernel lmad_copy_4b;+  gpu_kernel lmad_copy_8b;+};++struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx) {+  struct builtin_kernels *kernels = malloc(sizeof(struct builtin_kernels));+  gpu_create_kernel(ctx, &kernels->map_transpose_1b, "map_transpose_1b");+  gpu_create_kernel(ctx, &kernels->map_transpose_1b_large, "map_transpose_1b_large");+  gpu_create_kernel(ctx, &kernels->map_transpose_1b_low_height, "map_transpose_1b_low_height");+  gpu_create_kernel(ctx, &kernels->map_transpose_1b_low_width, "map_transpose_1b_low_width");+  gpu_create_kernel(ctx, &kernels->map_transpose_1b_small, "map_transpose_1b_small");++  gpu_create_kernel(ctx, &kernels->map_transpose_2b, "map_transpose_2b");+  gpu_create_kernel(ctx, &kernels->map_transpose_2b_large, "map_transpose_2b_large");+  gpu_create_kernel(ctx, &kernels->map_transpose_2b_low_height, "map_transpose_2b_low_height");+  gpu_create_kernel(ctx, &kernels->map_transpose_2b_low_width, "map_transpose_2b_low_width");+  gpu_create_kernel(ctx, &kernels->map_transpose_2b_small, "map_transpose_2b_small");++  gpu_create_kernel(ctx, &kernels->map_transpose_4b, "map_transpose_4b");+  gpu_create_kernel(ctx, &kernels->map_transpose_4b_large, "map_transpose_4b_large");+  gpu_create_kernel(ctx, &kernels->map_transpose_4b_low_height, "map_transpose_4b_low_height");+  gpu_create_kernel(ctx, &kernels->map_transpose_4b_low_width, "map_transpose_4b_low_width");+  gpu_create_kernel(ctx, &kernels->map_transpose_4b_small, "map_transpose_4b_small");++  gpu_create_kernel(ctx, &kernels->map_transpose_8b, "map_transpose_8b");+  gpu_create_kernel(ctx, &kernels->map_transpose_8b_large, "map_transpose_8b_large");+  gpu_create_kernel(ctx, &kernels->map_transpose_8b_low_height, "map_transpose_8b_low_height");+  gpu_create_kernel(ctx, &kernels->map_transpose_8b_low_width, "map_transpose_8b_low_width");+  gpu_create_kernel(ctx, &kernels->map_transpose_8b_small, "map_transpose_8b_small");++  gpu_create_kernel(ctx, &kernels->lmad_copy_1b, "lmad_copy_1b");+  gpu_create_kernel(ctx, &kernels->lmad_copy_2b, "lmad_copy_2b");+  gpu_create_kernel(ctx, &kernels->lmad_copy_4b, "lmad_copy_4b");+  gpu_create_kernel(ctx, &kernels->lmad_copy_8b, "lmad_copy_8b");++  return kernels;+}++void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels) {+  gpu_free_kernel(ctx, kernels->map_transpose_1b);+  gpu_free_kernel(ctx, kernels->map_transpose_1b_large);+  gpu_free_kernel(ctx, kernels->map_transpose_1b_low_height);+  gpu_free_kernel(ctx, kernels->map_transpose_1b_low_width);+  gpu_free_kernel(ctx, kernels->map_transpose_1b_small);++  gpu_free_kernel(ctx, kernels->map_transpose_2b);+  gpu_free_kernel(ctx, kernels->map_transpose_2b_large);+  gpu_free_kernel(ctx, kernels->map_transpose_2b_low_height);+  gpu_free_kernel(ctx, kernels->map_transpose_2b_low_width);+  gpu_free_kernel(ctx, kernels->map_transpose_2b_small);++  gpu_free_kernel(ctx, kernels->map_transpose_4b);+  gpu_free_kernel(ctx, kernels->map_transpose_4b_large);+  gpu_free_kernel(ctx, kernels->map_transpose_4b_low_height);+  gpu_free_kernel(ctx, kernels->map_transpose_4b_low_width);+  gpu_free_kernel(ctx, kernels->map_transpose_4b_small);++  gpu_free_kernel(ctx, kernels->map_transpose_8b);+  gpu_free_kernel(ctx, kernels->map_transpose_8b_large);+  gpu_free_kernel(ctx, kernels->map_transpose_8b_low_height);+  gpu_free_kernel(ctx, kernels->map_transpose_8b_low_width);+  gpu_free_kernel(ctx, kernels->map_transpose_8b_small);++  gpu_free_kernel(ctx, kernels->lmad_copy_1b);+  gpu_free_kernel(ctx, kernels->lmad_copy_2b);+  gpu_free_kernel(ctx, kernels->lmad_copy_4b);+  gpu_free_kernel(ctx, kernels->lmad_copy_8b);++  free(kernels);+}++static int gpu_alloc(struct futhark_context *ctx, FILE *log,+                     size_t min_size, const char *tag,+                     gpu_mem *mem_out, size_t *size_out) {+  if (min_size < sizeof(int)) {+    min_size = sizeof(int);+  }++  gpu_mem* memptr;+  if (free_list_find(&ctx->gpu_free_list, min_size, tag, size_out, (fl_mem*)&memptr) == 0) {+    // Successfully found a free block.  Is it big enough?+    if (*size_out >= min_size) {+      if (ctx->cfg->debugging) {+        fprintf(log, "No need to allocate: Found a block in the free list.\n");+      }+      *mem_out = *memptr;+      free(memptr);+      return FUTHARK_SUCCESS;+    } else {+      if (ctx->cfg->debugging) {+        fprintf(log, "Found a free block, but it was too small.\n");+      }+      int error = gpu_free_actual(ctx, *memptr);+      free(memptr);+      if (error != FUTHARK_SUCCESS) {+        return error;+      }+    }+  }++  *size_out = min_size;++  // We have to allocate a new block from the driver.  If the+  // allocation does not succeed, then we might be in an out-of-memory+  // situation.  We now start freeing things from the free list until+  // we think we have freed enough that the allocation will succeed.+  // Since we don't know how far the allocation is from fitting, we+  // have to check after every deallocation.  This might be pretty+  // expensive.  Let's hope that this case is hit rarely.++  if (ctx->cfg->debugging) {+    fprintf(log, "Actually allocating the desired block.\n");+  }++  int error = gpu_alloc_actual(ctx, min_size, mem_out);++  while (error == FUTHARK_OUT_OF_MEMORY) {+    if (ctx->cfg->debugging) {+      fprintf(log, "Out of GPU memory: releasing entry from the free list...\n");+    }+    gpu_mem* memptr;+    if (free_list_first(&ctx->gpu_free_list, (fl_mem*)&memptr) == 0) {+      gpu_mem mem = *memptr;+      free(memptr);+      error = gpu_free_actual(ctx, mem);+      if (error != FUTHARK_SUCCESS) {+        return error;+      }+    } else {+      break;+    }+    error = gpu_alloc_actual(ctx, min_size, mem_out);+  }++  return error;+}++static int gpu_free(struct futhark_context *ctx,+                    gpu_mem mem, size_t size, const char *tag) {+  gpu_mem* memptr = malloc(sizeof(gpu_mem));+  *memptr = mem;+  free_list_insert(&ctx->gpu_free_list, size, (fl_mem)memptr, tag);+  return FUTHARK_SUCCESS;+}++static int gpu_free_all(struct futhark_context *ctx) {+  free_list_pack(&ctx->gpu_free_list);+  gpu_mem* memptr;+  while (free_list_first(&ctx->gpu_free_list, (fl_mem*)&memptr) == 0) {+    gpu_mem mem = *memptr;+    free(memptr);+    int error = gpu_free_actual(ctx, mem);+    if (error != FUTHARK_SUCCESS) {+      return error;+    }+  }++  return FUTHARK_SUCCESS;+}++static int gpu_map_transpose(struct futhark_context* ctx,+                             gpu_kernel kernel_default,+                             gpu_kernel kernel_low_height,+                             gpu_kernel kernel_low_width,+                             gpu_kernel kernel_small,+                             gpu_kernel kernel_large,+                             const char *name, size_t elem_size,+                             gpu_mem dst, int64_t dst_offset,+                             gpu_mem src, int64_t src_offset,+                             int64_t k, int64_t n, int64_t m) {+  int64_t mulx = TR_BLOCK_DIM / n;+  int64_t muly = TR_BLOCK_DIM / m;+  int32_t mulx32 = mulx;+  int32_t muly32 = muly;+  int32_t k32 = k;+  int32_t n32 = n;+  int32_t m32 = m;++  gpu_kernel kernel = kernel_default;+  int32_t grid[3];+  int32_t block[3];++  void* args[11];+  size_t args_sizes[11] = {+    sizeof(gpu_mem), sizeof(int64_t),+    sizeof(gpu_mem), sizeof(int64_t),+    sizeof(int32_t),+    sizeof(int32_t),+    sizeof(int32_t),+    sizeof(int32_t),+    sizeof(int32_t)+  };++  args[0] = &dst;+  args[1] = &dst_offset;+  args[2] = &src;+  args[3] = &src_offset;+  args[7] = &mulx;+  args[8] = &muly;++  if (dst_offset + k * n * m <= 2147483647L &&+      src_offset + k * n * m <= 2147483647L) {+    if (m <= TR_BLOCK_DIM/2 && n <= TR_BLOCK_DIM/2) {+      if (ctx->logging) { fprintf(ctx->log, "Using small kernel\n"); }+      kernel = kernel_small;+      grid[0] = ((k * n * m) + (TR_BLOCK_DIM*TR_BLOCK_DIM) - 1) / (TR_BLOCK_DIM*TR_BLOCK_DIM);+      grid[1] = 1;+      grid[2] = 1;+      block[0] = TR_BLOCK_DIM*TR_BLOCK_DIM;+      block[1] = 1;+      block[2] = 1;+    } else if (m <= TR_BLOCK_DIM/2 && TR_BLOCK_DIM < n) {+      if (ctx->logging) { fprintf(ctx->log, "Using low-width kernel\n"); }+      kernel = kernel_low_width;+      int64_t x_elems = m;+      int64_t y_elems = (n + muly - 1) / muly;+      grid[0] = (x_elems + TR_BLOCK_DIM - 1) / TR_BLOCK_DIM;+      grid[1] = (y_elems + TR_BLOCK_DIM - 1) / TR_BLOCK_DIM;+      grid[2] = k;+      block[0] = TR_BLOCK_DIM;+      block[1] = TR_BLOCK_DIM;+      block[2] = 1;+    } else if (n <= TR_BLOCK_DIM/2 && TR_BLOCK_DIM < m) {+      if (ctx->logging) { fprintf(ctx->log, "Using low-height kernel\n"); }+      kernel = kernel_low_height;+      int64_t x_elems = (m + mulx - 1) / mulx;+      int64_t y_elems = n;+      grid[0] = (x_elems + TR_BLOCK_DIM - 1) / TR_BLOCK_DIM;+      grid[1] = (y_elems + TR_BLOCK_DIM - 1) / TR_BLOCK_DIM;+      grid[2] = k;+      block[0] = TR_BLOCK_DIM;+      block[1] = TR_BLOCK_DIM;+      block[2] = 1;+    } else {+      if (ctx->logging) { fprintf(ctx->log, "Using default kernel\n"); }+      kernel = kernel_default;+      grid[0] = (m+TR_TILE_DIM-1)/TR_TILE_DIM;+      grid[1] = (n+TR_TILE_DIM-1)/TR_TILE_DIM;+      grid[2] = k;+      block[0] = TR_TILE_DIM;+      block[1] = TR_TILE_DIM/TR_ELEMS_PER_THREAD;+      block[2] = 1;+    }+    args[4] = &k32;+    args[5] = &m32;+    args[6] = &n32;+    args[7] = &mulx32;+    args[8] = &muly32;+  } else {+    if (ctx->logging) { fprintf(ctx->log, "Using large kernel\n"); }+    kernel = kernel_large;+    grid[0] = (m+TR_TILE_DIM-1)/TR_TILE_DIM;+    grid[1] = (n+TR_TILE_DIM-1)/TR_TILE_DIM;+    grid[2] = k;+    block[0] = TR_TILE_DIM;+    block[1] = TR_TILE_DIM/TR_ELEMS_PER_THREAD;+    block[2] = 1;+    args[4] = &k;+    args[5] = &m;+    args[6] = &n;+    args[7] = &mulx;+    args[8] = &muly;+    args_sizes[4] = sizeof(int64_t);+    args_sizes[5] = sizeof(int64_t);+    args_sizes[6] = sizeof(int64_t);+    args_sizes[7] = sizeof(int64_t);+    args_sizes[8] = sizeof(int64_t);+  }++  // Cap the number of groups we launch and figure out how many+  // repeats we need alongside each dimension.+  int32_t repeat_1 = grid[1] / MAX_TR_GROUPS;+  int32_t repeat_2 = grid[2] / MAX_TR_GROUPS;+  grid[1] = repeat_1 > 0 ? MAX_TR_GROUPS : grid[1];+  grid[2] = repeat_2 > 0 ? MAX_TR_GROUPS : grid[2];+  args[9] = &repeat_1;+  args[10] = &repeat_2;+  args_sizes[9] = sizeof(repeat_1);+  args_sizes[10] = sizeof(repeat_2);++  if (ctx->logging) {+    fprintf(ctx->log, "\n");+  }++  return gpu_launch_kernel(ctx, kernel, name, grid, block,+                           TR_TILE_DIM*(TR_TILE_DIM+1)*elem_size,+                           sizeof(args)/sizeof(args[0]), args, args_sizes);+}++#define GEN_MAP_TRANSPOSE_GPU2GPU(NAME, ELEM_TYPE)                      \+  static int map_transpose_gpu2gpu_##NAME                               \+  (struct futhark_context* ctx,                                         \+   gpu_mem dst, int64_t dst_offset,                                     \+   gpu_mem src, int64_t src_offset,                                     \+   int64_t k, int64_t m, int64_t n)                                     \+  {                                                                     \+    return                                                              \+      gpu_map_transpose                                                 \+      (ctx,                                                             \+       ctx->kernels->map_transpose_##NAME,                              \+       ctx->kernels->map_transpose_##NAME##_low_height,                 \+       ctx->kernels->map_transpose_##NAME##_low_width,                  \+       ctx->kernels->map_transpose_##NAME##_small,                      \+       ctx->kernels->map_transpose_##NAME##_large,                      \+       "map_transpose_" #NAME, sizeof(ELEM_TYPE),                       \+       dst, dst_offset, src, src_offset,                                \+       k, n, m);                                                        \+  }++static int gpu_lmad_copy(struct futhark_context* ctx,+                         gpu_kernel kernel, int r,+                         gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],+                         gpu_mem src, int64_t src_offset, int64_t src_strides[r],+                         int64_t shape[r]) {+  if (r > 8) {+    set_error(ctx, strdup("Futhark runtime limitation:\nCannot copy array of greater than rank 8.\n"));+    return 1;+  }++  int64_t n = 1;+  for (int i = 0; i < r; i++) { n *= shape[i]; }++  void* args[6+(8*3)];+  size_t args_sizes[6+(8*3)];++  args[0] = &dst;+  args_sizes[0] = sizeof(gpu_mem);+  args[1] = &dst_offset;+  args_sizes[1] = sizeof(dst_offset);+  args[2] = &src;+  args_sizes[2] = sizeof(gpu_mem);+  args[3] = &src_offset;+  args_sizes[3] = sizeof(src_offset);+  args[4] = &n;+  args_sizes[4] = sizeof(n);+  args[5] = &r;+  args_sizes[5] = sizeof(r);++  int64_t zero = 0;++  for (int i = 0; i < 8; i++) {+    args_sizes[6+i*3] = sizeof(int64_t);+    args_sizes[6+i*3+1] = sizeof(int64_t);+    args_sizes[6+i*3+2] = sizeof(int64_t);+    if (i < r) {+      args[6+i*3] = &shape[i];+      args[6+i*3+1] = &dst_strides[i];+      args[6+i*3+2] = &src_strides[i];+    } else {+      args[6+i*3] = &zero;+      args[6+i*3+1] = &zero;+      args[6+i*3+2] = &zero;+    }+  }+  const size_t w = 256; // XXX: hardcoded workgroup size.++  return gpu_launch_kernel(ctx, kernel, "copy_lmad_dev_to_dev",+                           (const int32_t[3]) {(n+w-1)/w,1,1},+                           (const int32_t[3]) {w,1,1},+                           0, 6+(8*3), args, args_sizes);+}++#define GEN_LMAD_COPY_ELEMENTS_GPU2GPU(NAME, ELEM_TYPE)                 \+  static int lmad_copy_elements_gpu2gpu_##NAME                          \+  (struct futhark_context* ctx,                                         \+   int r,                                                               \+   gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],             \+   gpu_mem src, int64_t src_offset, int64_t src_strides[r],             \+   int64_t shape[r]) {                                                  \+    return gpu_lmad_copy(ctx, ctx->kernels->lmad_copy_##NAME, r,        \+                         dst, dst_offset, dst_strides,                  \+                         src, src_offset, src_strides,                  \+                         shape);                                        \+  }                                                                     \++#define GEN_LMAD_COPY_GPU2GPU(NAME, ELEM_TYPE)                          \+  static int lmad_copy_gpu2gpu_##NAME                                   \+  (struct futhark_context* ctx,                                         \+   int r,                                                               \+   gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],             \+   gpu_mem src, int64_t src_offset, int64_t src_strides[r],             \+   int64_t shape[r]) {                                                  \+    log_copy(ctx, "GPU to GPU", r, dst_offset, dst_strides,             \+             src_offset, src_strides, shape);                           \+    int64_t size = 1;                                                   \+    for (int i = 0; i < r; i++) { size *= shape[i]; }                   \+    if (size == 0) { return FUTHARK_SUCCESS; }                          \+    int64_t k, n, m;                                                    \+    if (lmad_map_tr(&k, &n, &m,                                         \+                       r, dst_strides, src_strides, shape)) {           \+      log_transpose(ctx, k, n, m);                                      \+      return map_transpose_gpu2gpu_##NAME                               \+        (ctx, dst, dst_offset, src, src_offset, k, n, m);               \+    } else if (lmad_memcpyable(r, dst_strides, src_strides, shape)) {   \+      if (ctx->logging) {fprintf(ctx->log, "## Flat copy\n\n");}        \+      return gpu_memcpy(ctx,                                            \+                        dst, dst_offset*sizeof(ELEM_TYPE),              \+                        src, src_offset*sizeof(ELEM_TYPE),              \+                        size * sizeof(ELEM_TYPE));                      \+    } else {                                                            \+      if (ctx->logging) {fprintf(ctx->log, "## General copy\n\n");}     \+      return lmad_copy_elements_gpu2gpu_##NAME                          \+        (ctx, r,                                                        \+         dst, dst_offset, dst_strides,                                  \+         src, src_offset, src_strides,                                  \+         shape);                                                        \+    }                                                                   \+  }++static int+lmad_copy_elements_host2gpu(struct futhark_context *ctx, size_t elem_size,+                            int r,+                            gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],+                            unsigned char* src, int64_t src_offset, int64_t src_strides[r],+                            int64_t shape[r]) {+  (void)ctx; (void)elem_size; (void)r;+  (void)dst; (void)dst_offset; (void)dst_strides;+  (void)src; (void)src_offset; (void)src_strides;+  (void)shape;+  set_error(ctx, strdup("Futhark runtime limitation:\nCannot copy unstructured array from host to GPU.\n"));+  return 1;+}++static int+lmad_copy_elements_gpu2host (struct futhark_context *ctx, size_t elem_size,+                             int r,+                             unsigned char* dst, int64_t dst_offset, int64_t dst_strides[r],+                             gpu_mem src, int64_t src_offset, int64_t src_strides[r],+                             int64_t shape[r]) {+  (void)ctx; (void)elem_size; (void)r;+  (void)dst; (void)dst_offset; (void)dst_strides;+  (void)src; (void)src_offset; (void)src_strides;+  (void)shape;+  set_error(ctx, strdup("Futhark runtime limitation:\nCannot copy unstructured array from GPU to host.\n"));+  return 1;+}++#define GEN_LMAD_COPY_ELEMENTS_HOSTGPU(NAME, ELEM_TYPE)                 \+  static int lmad_copy_elements_gpu2gpu_##NAME                          \+  (struct futhark_context* ctx,                                         \+   int r,                                                               \+   gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],             \+   gpu_mem src, int64_t src_offset, int64_t src_strides[r],             \+   int64_t shape[r]) {                                                  \+    return (ctx, ctx->kernels->lmad_copy_##NAME, r,                     \+                         dst, dst_offset, dst_strides,                  \+                         src, src_offset, src_strides,                  \+                         shape);                                        \+  }                                                                     \+++static int lmad_copy_host2gpu(struct futhark_context* ctx, size_t elem_size, bool sync,+                              int r,+                              gpu_mem dst, int64_t dst_offset, int64_t dst_strides[r],+                              unsigned char* src, int64_t src_offset, int64_t src_strides[r],+                              int64_t shape[r]) {+  log_copy(ctx, "Host to GPU", r, dst_offset, dst_strides,+           src_offset, src_strides, shape);+  int64_t size = elem_size;+  for (int i = 0; i < r; i++) { size *= shape[i]; }+  if (size == 0) { return FUTHARK_SUCCESS; }+  int64_t k, n, m;+  if (lmad_memcpyable(r, dst_strides, src_strides, shape)) {+    if (ctx->logging) {fprintf(ctx->log, "## Flat copy\n\n");}+    return memcpy_host2gpu(ctx, sync,+                           dst, dst_offset*elem_size,+                           src, src_offset*elem_size,+                           size);+  } else {+    if (ctx->logging) {fprintf(ctx->log, "## General copy\n\n");}+    int error;+    error = lmad_copy_elements_host2gpu+      (ctx, elem_size, r,+       dst, dst_offset, dst_strides,+       src, src_offset, src_strides,+       shape);+    if (error == 0 && sync) {+      error = futhark_context_sync(ctx);+    }+    return error;+  }+}++static int lmad_copy_gpu2host(struct futhark_context* ctx, size_t elem_size, bool sync,+                              int r,+                              unsigned char* dst, int64_t dst_offset, int64_t dst_strides[r],+                              gpu_mem src, int64_t src_offset, int64_t src_strides[r],+                              int64_t shape[r]) {+  log_copy(ctx, "Host to GPU", r, dst_offset, dst_strides,+           src_offset, src_strides, shape);+  int64_t size = elem_size;+  for (int i = 0; i < r; i++) { size *= shape[i]; }+  if (size == 0) { return FUTHARK_SUCCESS; }+  int64_t k, n, m;+  if (lmad_memcpyable(r, dst_strides, src_strides, shape)) {+    if (ctx->logging) {fprintf(ctx->log, "## Flat copy\n\n");}+    return memcpy_gpu2host(ctx, sync,+                           dst, dst_offset*elem_size,+                           src, src_offset*elem_size,+                           size);+  } else {+    if (ctx->logging) {fprintf(ctx->log, "## General copy\n\n");}+    int error;+    error = lmad_copy_elements_gpu2host+      (ctx, elem_size, r,+       dst, dst_offset, dst_strides,+       src, src_offset, src_strides,+       shape);+    if (error == 0 && sync) {+      error = futhark_context_sync(ctx);+    }+    return error;+  }+}++GEN_MAP_TRANSPOSE_GPU2GPU(1b, uint8_t)+GEN_MAP_TRANSPOSE_GPU2GPU(2b, uint16_t)+GEN_MAP_TRANSPOSE_GPU2GPU(4b, uint32_t)+GEN_MAP_TRANSPOSE_GPU2GPU(8b, uint64_t)++GEN_LMAD_COPY_ELEMENTS_GPU2GPU(1b, uint8_t)+GEN_LMAD_COPY_ELEMENTS_GPU2GPU(2b, uint16_t)+GEN_LMAD_COPY_ELEMENTS_GPU2GPU(4b, uint32_t)+GEN_LMAD_COPY_ELEMENTS_GPU2GPU(8b, uint64_t)++GEN_LMAD_COPY_GPU2GPU(1b, uint8_t)+GEN_LMAD_COPY_GPU2GPU(2b, uint16_t)+GEN_LMAD_COPY_GPU2GPU(4b, uint32_t)+GEN_LMAD_COPY_GPU2GPU(8b, uint64_t)++// End of gpu.h

@@ -0,0 +1,12 @@+// Start of gpu_prototypes.h++// Constants used for transpositions.  In principle these should be configurable.+#define TR_BLOCK_DIM 16+#define TR_TILE_DIM (TR_BLOCK_DIM*2)+#define TR_ELEMS_PER_THREAD 8++struct builtin_kernels* init_builtin_kernels(struct futhark_context* ctx);+void free_builtin_kernels(struct futhark_context* ctx, struct builtin_kernels* kernels);+static int gpu_free_all(struct futhark_context *ctx);++// End of gpu_prototypes.h

@@ -217,7 +217,7 @@   0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024,   0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; -static uint16_t float2halfbits(float value) {+SCALAR_FUN_ATTR uint16_t float2halfbits(float value) {   union { float x; uint32_t y; } u;   u.x = value;   uint32_t bits = u.y;@@ -227,7 +227,7 @@   return hbits; } -static float halfbits2float(uint16_t value) {+SCALAR_FUN_ATTR float halfbits2float(uint16_t value) {   uint32_t bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10];    union { uint32_t x; float y; } u;@@ -235,7 +235,7 @@   return u.y; } -static uint16_t halfbitsnextafter(uint16_t from, uint16_t to) {+SCALAR_FUN_ATTR uint16_t halfbitsnextafter(uint16_t from, uint16_t to) {   int fabs = from & 0x7FFF, tabs = to & 0x7FFF;   if(fabs > 0x7C00 || tabs > 0x7C00) {     return ((from&0x7FFF)>0x7C00) ? (from|0x200) : (to|0x200);

@@ -399,58 +399,4 @@   return err; } -// AOS <-> SOA memcpy functions-#define memmove_sized(dim)                                                                                      \-static inline void memmove_##dim(varying uint8 * uniform dst, uniform uint8 * varying src, uniform int64_t n) { \-    uniform uint##dim * varying srcp = (uniform uint##dim * varying) src;                                       \-    varying uint##dim * uniform dstp = (varying uint##dim * uniform) dst;                                       \-    for (uniform int64_t i = 0; i < n / (dim / 8); i++) {                                                       \-        dstp[i] = srcp[i];                                                                                      \-    }                                                                                                           \-}                                                                                                               \-static inline void memmove_##dim(uniform uint8 * varying dst, varying uint8 * uniform src, uniform int64_t n) { \-    varying uint##dim * uniform srcp = (varying uint##dim * uniform) src;                                       \-    uniform uint##dim * varying dstp = (uniform uint##dim * varying) dst;                                       \-    for (uniform int64_t i = 0; i < n / (dim / 8); i++) {                                                       \-        dstp[i] = srcp[i];                                                                                      \-    }                                                                                                           \-}                                                                                                               \-static inline void memmove_##dim(varying uint8 * uniform dst, varying uint8 * uniform src, uniform int64_t n) { \-    varying uint##dim * uniform srcp = (varying uint##dim * uniform) src;                                       \-    varying uint##dim * uniform dstp = (varying uint##dim * uniform) dst;                                       \-    for (uniform int64_t i = 0; i < n / (dim / 8); i++) {                                                       \-        dstp[i] = srcp[i];                                                                                      \-    }                                                                                                           \-}                                                                                                               \-static inline void memmove_##dim(varying uint8 * varying dst, uniform uint8 * varying src, uniform int64_t n) { \-    foreach_unique (ptr in dst) {                                                                               \-        memmove_##dim(ptr, src, n);                                                                             \-    }                                                                                                           \-}                                                                                                               \-static inline void memmove_##dim(uniform uint8 * varying dst, varying uint8 * varying src, uniform int64_t n) { \-    foreach_unique (ptr in src) {                                                                               \-        memmove_##dim(dst, ptr, n);                                                                             \-    }                                                                                                           \-}                                                                                                               \-static inline void memmove_##dim(varying uint8 * varying dst, varying uint8 * uniform src, uniform int64_t n) { \-    foreach_unique (ptr in dst) {                                                                               \-        memmove_##dim(ptr, src, n);                                                                             \-    }                                                                                                           \-}                                                                                                               \-static inline void memmove_##dim(varying uint8 * varying dst, varying uint8 * varying src, uniform int64_t n) { \-    if (reduce_equal((varying int64_t)dst)) {                                                                   \-        foreach_unique (ptr in src) {                                                                           \-            memmove_##dim(dst, ptr, n);                                                                         \-        }                                                                                                       \-    } else {                                                                                                    \-        foreach_unique (ptr in dst) {                                                                           \-            memmove_##dim(ptr, src, n);                                                                         \-        }                                                                                                       \-    }                                                                                                           \-}-memmove_sized(8)-memmove_sized(16)-memmove_sized(32)-memmove_sized(64)- // End of ispc_util.h.

@@ -17,2980 +17,2980 @@ // Double-precision definitions are only included if the preprocessor // macro FUTHARK_F64_ENABLED is set. -static inline uint8_t add8(uint8_t x, uint8_t y) {-  return x + y;-}--static inline uint16_t add16(uint16_t x, uint16_t y) {-  return x + y;-}--static inline uint32_t add32(uint32_t x, uint32_t y) {-  return x + y;-}--static inline uint64_t add64(uint64_t x, uint64_t y) {-  return x + y;-}--static inline uint8_t sub8(uint8_t x, uint8_t y) {-  return x - y;-}--static inline uint16_t sub16(uint16_t x, uint16_t y) {-  return x - y;-}--static inline uint32_t sub32(uint32_t x, uint32_t y) {-  return x - y;-}--static inline uint64_t sub64(uint64_t x, uint64_t y) {-  return x - y;-}--static inline uint8_t mul8(uint8_t x, uint8_t y) {-  return x * y;-}--static inline uint16_t mul16(uint16_t x, uint16_t y) {-  return x * y;-}--static inline uint32_t mul32(uint32_t x, uint32_t y) {-  return x * y;-}--static inline uint64_t mul64(uint64_t x, uint64_t y) {-  return x * y;-}--#if ISPC--static inline uint8_t udiv8(uint8_t x, uint8_t y) {-  // This strange pattern is used to prevent the ISPC compiler from-  // causing SIGFPEs and bogus results on divisions where inactive lanes-  // have 0-valued divisors. It ensures that any inactive lane instead-  // has a divisor of 1. https://github.com/ispc/ispc/issues/2292-  uint8_t ys = 1;-  foreach_active(i){-    ys = y;-  }--  return x / ys;-}--static inline uint16_t udiv16(uint16_t x, uint16_t y) {-  uint16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x / ys;-}--static inline uint32_t udiv32(uint32_t x, uint32_t y) {-  uint32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  --  return x / ys;-}--static inline uint64_t udiv64(uint64_t x, uint64_t y) {-  uint64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  --  return x / ys;-}--static inline uint8_t udiv_up8(uint8_t x, uint8_t y) {-  uint8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  --  return (x + y - 1) / ys;-}--static inline uint16_t udiv_up16(uint16_t x, uint16_t y) {-  uint16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return (x + y - 1) / ys;-}--static inline uint32_t udiv_up32(uint32_t x, uint32_t y) {-  uint32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return (x + y - 1) / ys;-}--static inline uint64_t udiv_up64(uint64_t x, uint64_t y) {-  uint64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return (x + y - 1) / ys;-}--static inline uint8_t umod8(uint8_t x, uint8_t y) {-  uint8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x % ys;-}--static inline uint16_t umod16(uint16_t x, uint16_t y) {-  uint16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  --  return x % ys;-}--static inline uint32_t umod32(uint32_t x, uint32_t y) {-  uint32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x % ys;-}--static inline uint64_t umod64(uint64_t x, uint64_t y) {-  uint64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x % ys;-}--static inline uint8_t udiv_safe8(uint8_t x, uint8_t y) {-  uint8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x / ys;-}--static inline uint16_t udiv_safe16(uint16_t x, uint16_t y) {-  uint16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x / ys;-}--static inline uint32_t udiv_safe32(uint32_t x, uint32_t y) {-  uint32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x / ys;-}--static inline uint64_t udiv_safe64(uint64_t x, uint64_t y) {-  uint64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x / ys;-}--static inline uint8_t udiv_up_safe8(uint8_t x, uint8_t y) {-  uint8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : (x + y - 1) / ys;-}--static inline uint16_t udiv_up_safe16(uint16_t x, uint16_t y) {-  uint16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : (x + y - 1) / ys;-}--static inline uint32_t udiv_up_safe32(uint32_t x, uint32_t y) {-  uint32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : (x + y - 1) / ys;-}--static inline uint64_t udiv_up_safe64(uint64_t x, uint64_t y) {-  uint64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : (x + y - 1) / ys;-}--static inline uint8_t umod_safe8(uint8_t x, uint8_t y) {-  uint8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x % ys;-}--static inline uint16_t umod_safe16(uint16_t x, uint16_t y) {-  uint16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x % ys;-}--static inline uint32_t umod_safe32(uint32_t x, uint32_t y) {-  uint32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x % ys;-}--static inline uint64_t umod_safe64(uint64_t x, uint64_t y) {-  uint64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x % ys;-}--static inline int8_t sdiv8(int8_t x, int8_t y) {-  int8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  int8_t q = x / ys;-  int8_t r = x % ys;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int16_t sdiv16(int16_t x, int16_t y) {-  int16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  int16_t q = x / ys;-  int16_t r = x % ys;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int32_t sdiv32(int32_t x, int32_t y) {-  int32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  int32_t q = x / ys;-  int32_t r = x % ys;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int64_t sdiv64(int64_t x, int64_t y) {-  int64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  int64_t q = x / ys;-  int64_t r = x % ys;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int8_t sdiv_up8(int8_t x, int8_t y) {-  return sdiv8(x + y - 1, y);-}--static inline int16_t sdiv_up16(int16_t x, int16_t y) {-  return sdiv16(x + y - 1, y);-}--static inline int32_t sdiv_up32(int32_t x, int32_t y) {-  return sdiv32(x + y - 1, y);-}--static inline int64_t sdiv_up64(int64_t x, int64_t y) {-  return sdiv64(x + y - 1, y);-}--static inline int8_t smod8(int8_t x, int8_t y) {-  int8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  int8_t r = x % ys;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int16_t smod16(int16_t x, int16_t y) {-  int16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  int16_t r = x % ys;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int32_t smod32(int32_t x, int32_t y) {-  int32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  int32_t r = x % ys;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int64_t smod64(int64_t x, int64_t y) {-  int64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  int64_t r = x % ys;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int8_t sdiv_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : sdiv8(x, y);-}--static inline int16_t sdiv_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : sdiv16(x, y);-}--static inline int32_t sdiv_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : sdiv32(x, y);-}--static inline int64_t sdiv_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : sdiv64(x, y);-}--static inline int8_t sdiv_up_safe8(int8_t x, int8_t y) {-  return sdiv_safe8(x + y - 1, y);-}--static inline int16_t sdiv_up_safe16(int16_t x, int16_t y) {-  return sdiv_safe16(x + y - 1, y);-}--static inline int32_t sdiv_up_safe32(int32_t x, int32_t y) {-  return sdiv_safe32(x + y - 1, y);-}--static inline int64_t sdiv_up_safe64(int64_t x, int64_t y) {-  return sdiv_safe64(x + y - 1, y);-}--static inline int8_t smod_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : smod8(x, y);-}--static inline int16_t smod_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : smod16(x, y);-}--static inline int32_t smod_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : smod32(x, y);-}--static inline int64_t smod_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : smod64(x, y);-}--static inline int8_t squot8(int8_t x, int8_t y) {-  int8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x / ys;-}--static inline int16_t squot16(int16_t x, int16_t y) {-  int16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x / ys;-}--static inline int32_t squot32(int32_t x, int32_t y) {-  int32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x / ys;-}--static inline int64_t squot64(int64_t x, int64_t y) {-  int64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x / ys;-}--static inline int8_t srem8(int8_t x, int8_t y) {-  int8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x % ys;-}--static inline int16_t srem16(int16_t x, int16_t y) {-  int16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x % ys;-}--static inline int32_t srem32(int32_t x, int32_t y) {-  int32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x % ys;-}--static inline int64_t srem64(int64_t x, int64_t y) {-  int8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return x % ys;-}--static inline int8_t squot_safe8(int8_t x, int8_t y) {-  int8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x / ys;-}--static inline int16_t squot_safe16(int16_t x, int16_t y) {-  int16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x / ys;-}--static inline int32_t squot_safe32(int32_t x, int32_t y) {-  int32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x / ys;-}--static inline int64_t squot_safe64(int64_t x, int64_t y) {-  int64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x / ys;-}--static inline int8_t srem_safe8(int8_t x, int8_t y) {-  int8_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x % ys;-}--static inline int16_t srem_safe16(int16_t x, int16_t y) {-  int16_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x % ys;-}--static inline int32_t srem_safe32(int32_t x, int32_t y) {-  int32_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x % ys;-}--static inline int64_t srem_safe64(int64_t x, int64_t y) {-  int64_t ys = 1;-  foreach_active(i){-    ys = y;-  }-  -  return y == 0 ? 0 : x % ys;-}--#else--static inline uint8_t udiv8(uint8_t x, uint8_t y) {-  return x / y;-}--static inline uint16_t udiv16(uint16_t x, uint16_t y) {-  return x / y;-}--static inline uint32_t udiv32(uint32_t x, uint32_t y) {-  return x / y;-}--static inline uint64_t udiv64(uint64_t x, uint64_t y) {-  return x / y;-}--static inline uint8_t udiv_up8(uint8_t x, uint8_t y) {-  return (x + y - 1) / y;-}--static inline uint16_t udiv_up16(uint16_t x, uint16_t y) {-  return (x + y - 1) / y;-}--static inline uint32_t udiv_up32(uint32_t x, uint32_t y) {-  return (x + y - 1) / y;-}--static inline uint64_t udiv_up64(uint64_t x, uint64_t y) {-  return (x + y - 1) / y;-}--static inline uint8_t umod8(uint8_t x, uint8_t y) {-  return x % y;-}--static inline uint16_t umod16(uint16_t x, uint16_t y) {-  return x % y;-}--static inline uint32_t umod32(uint32_t x, uint32_t y) {-  return x % y;-}--static inline uint64_t umod64(uint64_t x, uint64_t y) {-  return x % y;-}--static inline uint8_t udiv_safe8(uint8_t x, uint8_t y) {-  return y == 0 ? 0 : x / y;-}--static inline uint16_t udiv_safe16(uint16_t x, uint16_t y) {-  return y == 0 ? 0 : x / y;-}--static inline uint32_t udiv_safe32(uint32_t x, uint32_t y) {-  return y == 0 ? 0 : x / y;-}--static inline uint64_t udiv_safe64(uint64_t x, uint64_t y) {-  return y == 0 ? 0 : x / y;-}--static inline uint8_t udiv_up_safe8(uint8_t x, uint8_t y) {-  return y == 0 ? 0 : (x + y - 1) / y;-}--static inline uint16_t udiv_up_safe16(uint16_t x, uint16_t y) {-  return y == 0 ? 0 : (x + y - 1) / y;-}--static inline uint32_t udiv_up_safe32(uint32_t x, uint32_t y) {-  return y == 0 ? 0 : (x + y - 1) / y;-}--static inline uint64_t udiv_up_safe64(uint64_t x, uint64_t y) {-  return y == 0 ? 0 : (x + y - 1) / y;-}--static inline uint8_t umod_safe8(uint8_t x, uint8_t y) {-  return y == 0 ? 0 : x % y;-}--static inline uint16_t umod_safe16(uint16_t x, uint16_t y) {-  return y == 0 ? 0 : x % y;-}--static inline uint32_t umod_safe32(uint32_t x, uint32_t y) {-  return y == 0 ? 0 : x % y;-}--static inline uint64_t umod_safe64(uint64_t x, uint64_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int8_t sdiv8(int8_t x, int8_t y) {-  int8_t q = x / y;-  int8_t r = x % y;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int16_t sdiv16(int16_t x, int16_t y) {-  int16_t q = x / y;-  int16_t r = x % y;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int32_t sdiv32(int32_t x, int32_t y) {-  int32_t q = x / y;-  int32_t r = x % y;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int64_t sdiv64(int64_t x, int64_t y) {-  int64_t q = x / y;-  int64_t r = x % y;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int8_t sdiv_up8(int8_t x, int8_t y) {-  return sdiv8(x + y - 1, y);-}--static inline int16_t sdiv_up16(int16_t x, int16_t y) {-  return sdiv16(x + y - 1, y);-}--static inline int32_t sdiv_up32(int32_t x, int32_t y) {-  return sdiv32(x + y - 1, y);-}--static inline int64_t sdiv_up64(int64_t x, int64_t y) {-  return sdiv64(x + y - 1, y);-}--static inline int8_t smod8(int8_t x, int8_t y) {-  int8_t r = x % y;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int16_t smod16(int16_t x, int16_t y) {-  int16_t r = x % y;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int32_t smod32(int32_t x, int32_t y) {-  int32_t r = x % y;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int64_t smod64(int64_t x, int64_t y) {-  int64_t r = x % y;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int8_t sdiv_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : sdiv8(x, y);-}--static inline int16_t sdiv_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : sdiv16(x, y);-}--static inline int32_t sdiv_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : sdiv32(x, y);-}--static inline int64_t sdiv_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : sdiv64(x, y);-}--static inline int8_t sdiv_up_safe8(int8_t x, int8_t y) {-  return sdiv_safe8(x + y - 1, y);-}--static inline int16_t sdiv_up_safe16(int16_t x, int16_t y) {-  return sdiv_safe16(x + y - 1, y);-}--static inline int32_t sdiv_up_safe32(int32_t x, int32_t y) {-  return sdiv_safe32(x + y - 1, y);-}--static inline int64_t sdiv_up_safe64(int64_t x, int64_t y) {-  return sdiv_safe64(x + y - 1, y);-}--static inline int8_t smod_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : smod8(x, y);-}--static inline int16_t smod_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : smod16(x, y);-}--static inline int32_t smod_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : smod32(x, y);-}--static inline int64_t smod_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : smod64(x, y);-}--static inline int8_t squot8(int8_t x, int8_t y) {-  return x / y;-}--static inline int16_t squot16(int16_t x, int16_t y) {-  return x / y;-}--static inline int32_t squot32(int32_t x, int32_t y) {-  return x / y;-}--static inline int64_t squot64(int64_t x, int64_t y) {-  return x / y;-}--static inline int8_t srem8(int8_t x, int8_t y) {-  return x % y;-}--static inline int16_t srem16(int16_t x, int16_t y) {-  return x % y;-}--static inline int32_t srem32(int32_t x, int32_t y) {-  return x % y;-}--static inline int64_t srem64(int64_t x, int64_t y) {-  return x % y;-}--static inline int8_t squot_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : x / y;-}--static inline int16_t squot_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : x / y;-}--static inline int32_t squot_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : x / y;-}--static inline int64_t squot_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : x / y;-}--static inline int8_t srem_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int16_t srem_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int32_t srem_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int64_t srem_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : x % y;-}--#endif--static inline int8_t smin8(int8_t x, int8_t y) {-  return x < y ? x : y;-}--static inline int16_t smin16(int16_t x, int16_t y) {-  return x < y ? x : y;-}--static inline int32_t smin32(int32_t x, int32_t y) {-  return x < y ? x : y;-}--static inline int64_t smin64(int64_t x, int64_t y) {-  return x < y ? x : y;-}--static inline uint8_t umin8(uint8_t x, uint8_t y) {-  return x < y ? x : y;-}--static inline uint16_t umin16(uint16_t x, uint16_t y) {-  return x < y ? x : y;-}--static inline uint32_t umin32(uint32_t x, uint32_t y) {-  return x < y ? x : y;-}--static inline uint64_t umin64(uint64_t x, uint64_t y) {-  return x < y ? x : y;-}--static inline int8_t smax8(int8_t x, int8_t y) {-  return x < y ? y : x;-}--static inline int16_t smax16(int16_t x, int16_t y) {-  return x < y ? y : x;-}--static inline int32_t smax32(int32_t x, int32_t y) {-  return x < y ? y : x;-}--static inline int64_t smax64(int64_t x, int64_t y) {-  return x < y ? y : x;-}--static inline uint8_t umax8(uint8_t x, uint8_t y) {-  return x < y ? y : x;-}--static inline uint16_t umax16(uint16_t x, uint16_t y) {-  return x < y ? y : x;-}--static inline uint32_t umax32(uint32_t x, uint32_t y) {-  return x < y ? y : x;-}--static inline uint64_t umax64(uint64_t x, uint64_t y) {-  return x < y ? y : x;-}--static inline uint8_t shl8(uint8_t x, uint8_t y) {-  return (uint8_t)(x << y);-}--static inline uint16_t shl16(uint16_t x, uint16_t y) {-  return (uint16_t)(x << y);-}--static inline uint32_t shl32(uint32_t x, uint32_t y) {-  return x << y;-}--static inline uint64_t shl64(uint64_t x, uint64_t y) {-  return x << y;-}--static inline uint8_t lshr8(uint8_t x, uint8_t y) {-  return x >> y;-}--static inline uint16_t lshr16(uint16_t x, uint16_t y) {-  return x >> y;-}--static inline uint32_t lshr32(uint32_t x, uint32_t y) {-  return x >> y;-}--static inline uint64_t lshr64(uint64_t x, uint64_t y) {-  return x >> y;-}--static inline int8_t ashr8(int8_t x, int8_t y) {-  return x >> y;-}--static inline int16_t ashr16(int16_t x, int16_t y) {-  return x >> y;-}--static inline int32_t ashr32(int32_t x, int32_t y) {-  return x >> y;-}--static inline int64_t ashr64(int64_t x, int64_t y) {-  return x >> y;-}--static inline uint8_t and8(uint8_t x, uint8_t y) {-  return x & y;-}--static inline uint16_t and16(uint16_t x, uint16_t y) {-  return x & y;-}--static inline uint32_t and32(uint32_t x, uint32_t y) {-  return x & y;-}--static inline uint64_t and64(uint64_t x, uint64_t y) {-  return x & y;-}--static inline uint8_t or8(uint8_t x, uint8_t y) {-  return x | y;-}--static inline uint16_t or16(uint16_t x, uint16_t y) {-  return x | y;-}--static inline uint32_t or32(uint32_t x, uint32_t y) {-  return x | y;-}--static inline uint64_t or64(uint64_t x, uint64_t y) {-  return x | y;-}--static inline uint8_t xor8(uint8_t x, uint8_t y) {-  return x ^ y;-}--static inline uint16_t xor16(uint16_t x, uint16_t y) {-  return x ^ y;-}--static inline uint32_t xor32(uint32_t x, uint32_t y) {-  return x ^ y;-}--static inline uint64_t xor64(uint64_t x, uint64_t y) {-  return x ^ y;-}--static inline bool ult8(uint8_t x, uint8_t y) {-  return x < y;-}--static inline bool ult16(uint16_t x, uint16_t y) {-  return x < y;-}--static inline bool ult32(uint32_t x, uint32_t y) {-  return x < y;-}--static inline bool ult64(uint64_t x, uint64_t y) {-  return x < y;-}--static inline bool ule8(uint8_t x, uint8_t y) {-  return x <= y;-}--static inline bool ule16(uint16_t x, uint16_t y) {-  return x <= y;-}--static inline bool ule32(uint32_t x, uint32_t y) {-  return x <= y;-}--static inline bool ule64(uint64_t x, uint64_t y) {-  return x <= y;-}--static inline bool slt8(int8_t x, int8_t y) {-  return x < y;-}--static inline bool slt16(int16_t x, int16_t y) {-  return x < y;-}--static inline bool slt32(int32_t x, int32_t y) {-  return x < y;-}--static inline bool slt64(int64_t x, int64_t y) {-  return x < y;-}--static inline bool sle8(int8_t x, int8_t y) {-  return x <= y;-}--static inline bool sle16(int16_t x, int16_t y) {-  return x <= y;-}--static inline bool sle32(int32_t x, int32_t y) {-  return x <= y;-}--static inline bool sle64(int64_t x, int64_t y) {-  return x <= y;-}--static inline uint8_t pow8(uint8_t x, uint8_t y) {-  uint8_t res = 1, rem = y;--  while (rem != 0) {-    if (rem & 1)-      res *= x;-    rem >>= 1;-    x *= x;-  }-  return res;-}--static inline uint16_t pow16(uint16_t x, uint16_t y) {-  uint16_t res = 1, rem = y;--  while (rem != 0) {-    if (rem & 1)-      res *= x;-    rem >>= 1;-    x *= x;-  }-  return res;-}--static inline uint32_t pow32(uint32_t x, uint32_t y) {-  uint32_t res = 1, rem = y;--  while (rem != 0) {-    if (rem & 1)-      res *= x;-    rem >>= 1;-    x *= x;-  }-  return res;-}--static inline uint64_t pow64(uint64_t x, uint64_t y) {-  uint64_t res = 1, rem = y;--  while (rem != 0) {-    if (rem & 1)-      res *= x;-    rem >>= 1;-    x *= x;-  }-  return res;-}--static inline bool itob_i8_bool(int8_t x) {-  return x != 0;-}--static inline bool itob_i16_bool(int16_t x) {-  return x != 0;-}--static inline bool itob_i32_bool(int32_t x) {-  return x != 0;-}--static inline bool itob_i64_bool(int64_t x) {-  return x != 0;-}--static inline int8_t btoi_bool_i8(bool x) {-  return x;-}--static inline int16_t btoi_bool_i16(bool x) {-  return x;-}--static inline int32_t btoi_bool_i32(bool x) {-  return x;-}--static inline int64_t btoi_bool_i64(bool x) {-  return x;-}--#define sext_i8_i8(x) ((int8_t) (int8_t) (x))-#define sext_i8_i16(x) ((int16_t) (int8_t) (x))-#define sext_i8_i32(x) ((int32_t) (int8_t) (x))-#define sext_i8_i64(x) ((int64_t) (int8_t) (x))-#define sext_i16_i8(x) ((int8_t) (int16_t) (x))-#define sext_i16_i16(x) ((int16_t) (int16_t) (x))-#define sext_i16_i32(x) ((int32_t) (int16_t) (x))-#define sext_i16_i64(x) ((int64_t) (int16_t) (x))-#define sext_i32_i8(x) ((int8_t) (int32_t) (x))-#define sext_i32_i16(x) ((int16_t) (int32_t) (x))-#define sext_i32_i32(x) ((int32_t) (int32_t) (x))-#define sext_i32_i64(x) ((int64_t) (int32_t) (x))-#define sext_i64_i8(x) ((int8_t) (int64_t) (x))-#define sext_i64_i16(x) ((int16_t) (int64_t) (x))-#define sext_i64_i32(x) ((int32_t) (int64_t) (x))-#define sext_i64_i64(x) ((int64_t) (int64_t) (x))-#define zext_i8_i8(x) ((int8_t) (uint8_t) (x))-#define zext_i8_i16(x) ((int16_t) (uint8_t) (x))-#define zext_i8_i32(x) ((int32_t) (uint8_t) (x))-#define zext_i8_i64(x) ((int64_t) (uint8_t) (x))-#define zext_i16_i8(x) ((int8_t) (uint16_t) (x))-#define zext_i16_i16(x) ((int16_t) (uint16_t) (x))-#define zext_i16_i32(x) ((int32_t) (uint16_t) (x))-#define zext_i16_i64(x) ((int64_t) (uint16_t) (x))-#define zext_i32_i8(x) ((int8_t) (uint32_t) (x))-#define zext_i32_i16(x) ((int16_t) (uint32_t) (x))-#define zext_i32_i32(x) ((int32_t) (uint32_t) (x))-#define zext_i32_i64(x) ((int64_t) (uint32_t) (x))-#define zext_i64_i8(x) ((int8_t) (uint64_t) (x))-#define zext_i64_i16(x) ((int16_t) (uint64_t) (x))-#define zext_i64_i32(x) ((int32_t) (uint64_t) (x))-#define zext_i64_i64(x) ((int64_t) (uint64_t) (x))--static int8_t abs8(int8_t x) {-  return (int8_t)abs(x);-}--static int16_t abs16(int16_t x) {-  return (int16_t)abs(x);-}--static int32_t abs32(int32_t x) {-  return abs(x);-}--static int64_t abs64(int64_t x) {-#if defined(__OPENCL_VERSION__) || defined(ISPC)-  return abs(x);-#else-  return llabs(x);-#endif-}--#if defined(__OPENCL_VERSION__)-static int32_t futrts_popc8(int8_t x) {-  return popcount(x);-}--static int32_t futrts_popc16(int16_t x) {-  return popcount(x);-}--static int32_t futrts_popc32(int32_t x) {-  return popcount(x);-}--static int32_t futrts_popc64(int64_t x) {-  return popcount(x);-}-#elif defined(__CUDA_ARCH__)--static int32_t futrts_popc8(int8_t x) {-  return __popc(zext_i8_i32(x));-}--static int32_t futrts_popc16(int16_t x) {-  return __popc(zext_i16_i32(x));-}--static int32_t futrts_popc32(int32_t x) {-  return __popc(x);-}--static int32_t futrts_popc64(int64_t x) {-  return __popcll(x);-}--#else // Not OpenCL or CUDA, but plain C.--static int32_t futrts_popc8(uint8_t x) {-  int c = 0;-  for (; x; ++c) { x &= x - 1; }-  return c;-}--static int32_t futrts_popc16(uint16_t x) {-  int c = 0;-  for (; x; ++c) { x &= x - 1; }-  return c;-}--static int32_t futrts_popc32(uint32_t x) {-  int c = 0;-  for (; x; ++c) { x &= x - 1; }-  return c;-}--static int32_t futrts_popc64(uint64_t x) {-  int c = 0;-  for (; x; ++c) { x &= x - 1; }-  return c;-}-#endif--#if defined(__OPENCL_VERSION__)-static uint8_t  futrts_umul_hi8 ( uint8_t a,  uint8_t b) { return mul_hi(a, b); }-static uint16_t futrts_umul_hi16(uint16_t a, uint16_t b) { return mul_hi(a, b); }-static uint32_t futrts_umul_hi32(uint32_t a, uint32_t b) { return mul_hi(a, b); }-static uint64_t futrts_umul_hi64(uint64_t a, uint64_t b) { return mul_hi(a, b); }-static uint8_t  futrts_smul_hi8 ( int8_t a,  int8_t b) { return mul_hi(a, b); }-static uint16_t futrts_smul_hi16(int16_t a, int16_t b) { return mul_hi(a, b); }-static uint32_t futrts_smul_hi32(int32_t a, int32_t b) { return mul_hi(a, b); }-static uint64_t futrts_smul_hi64(int64_t a, int64_t b) { return mul_hi(a, b); }-#elif defined(__CUDA_ARCH__)-static  uint8_t futrts_umul_hi8(uint8_t a, uint8_t b) { return ((uint16_t)a) * ((uint16_t)b) >> 8; }-static uint16_t futrts_umul_hi16(uint16_t a, uint16_t b) { return ((uint32_t)a) * ((uint32_t)b) >> 16; }-static uint32_t futrts_umul_hi32(uint32_t a, uint32_t b) { return __umulhi(a, b); }-static uint64_t futrts_umul_hi64(uint64_t a, uint64_t b) { return __umul64hi(a, b); }-static  uint8_t futrts_smul_hi8 ( int8_t a, int8_t b) { return ((int16_t)a) * ((int16_t)b) >> 8; }-static uint16_t futrts_smul_hi16(int16_t a, int16_t b) { return ((int32_t)a) * ((int32_t)b) >> 16; }-static uint32_t futrts_smul_hi32(int32_t a, int32_t b) { return __mulhi(a, b); }-static uint64_t futrts_smul_hi64(int64_t a, int64_t b) { return __mul64hi(a, b); }-#elif ISPC-static uint8_t futrts_umul_hi8(uint8_t a, uint8_t b) { return ((uint16_t)a) * ((uint16_t)b) >> 8; }-static uint16_t futrts_umul_hi16(uint16_t a, uint16_t b) { return ((uint32_t)a) * ((uint32_t)b) >> 16; }-static uint32_t futrts_umul_hi32(uint32_t a, uint32_t b) { return ((uint64_t)a) * ((uint64_t)b) >> 32; }-static uint64_t futrts_umul_hi64(uint64_t a, uint64_t b) {-  uint64_t ah = a >> 32;-  uint64_t al = a & 0xffffffff;-  uint64_t bh = b >> 32;-  uint64_t bl = b & 0xffffffff;--  uint64_t p1 = al * bl;-  uint64_t p2 = al * bh;-  uint64_t p3 = ah * bl;-  uint64_t p4 = ah * bh;--  uint64_t p1h = p1 >> 32;-  uint64_t p2h = p2 >> 32;-  uint64_t p3h = p3 >> 32;-  uint64_t p2l = p2 & 0xffffffff;-  uint64_t p3l = p3 & 0xffffffff;--  uint64_t l = p1h + p2l + p3l;-  uint64_t m = (p2 >> 32) + (p3 >> 32);-  uint64_t h = (l >> 32) + m + p4;--  return h;-}-static  int8_t futrts_smul_hi8 ( int8_t a,  int8_t b) { return ((uint16_t)a) * ((uint16_t)b) >> 8; }-static int16_t futrts_smul_hi16(int16_t a, int16_t b) { return ((uint32_t)a) * ((uint32_t)b) >> 16; }-static int32_t futrts_smul_hi32(int32_t a, int32_t b) { return ((uint64_t)a) * ((uint64_t)b) >> 32; }-static int64_t futrts_smul_hi64(int64_t a, int64_t b) {-  uint64_t ah = a >> 32;-  uint64_t al = a & 0xffffffff;-  uint64_t bh = b >> 32;-  uint64_t bl = b & 0xffffffff;--  uint64_t p1 =  al * bl;-  int64_t  p2 = al * bh;-  int64_t  p3 = ah * bl;-  uint64_t p4 =  ah * bh;--  uint64_t p1h = p1 >> 32;-  uint64_t p2h = p2 >> 32;-  uint64_t p3h = p3 >> 32;-  uint64_t p2l = p2 & 0xffffffff;-  uint64_t p3l = p3 & 0xffffffff;--  uint64_t l = p1h + p2l + p3l;-  uint64_t m = (p2 >> 32) + (p3 >> 32);-  uint64_t h = (l >> 32) + m + p4;--  return h;-}--#else // Not OpenCL, ISPC, or CUDA, but plain C.-static uint8_t futrts_umul_hi8(uint8_t a, uint8_t b) { return ((uint16_t)a) * ((uint16_t)b) >> 8; }-static uint16_t futrts_umul_hi16(uint16_t a, uint16_t b) { return ((uint32_t)a) * ((uint32_t)b) >> 16; }-static uint32_t futrts_umul_hi32(uint32_t a, uint32_t b) { return ((uint64_t)a) * ((uint64_t)b) >> 32; }-static uint64_t futrts_umul_hi64(uint64_t a, uint64_t b) { return ((__uint128_t)a) * ((__uint128_t)b) >> 64; }-static int8_t futrts_smul_hi8(int8_t a, int8_t b) { return ((int16_t)a) * ((int16_t)b) >> 8; }-static int16_t futrts_smul_hi16(int16_t a, int16_t b) { return ((int32_t)a) * ((int32_t)b) >> 16; }-static int32_t futrts_smul_hi32(int32_t a, int32_t b) { return ((int64_t)a) * ((int64_t)b) >> 32; }-static int64_t futrts_smul_hi64(int64_t a, int64_t b) { return ((__int128_t)a) * ((__int128_t)b) >> 64; }-#endif--#if defined(__OPENCL_VERSION__)-static  uint8_t futrts_umad_hi8 ( uint8_t a,  uint8_t b,  uint8_t c) { return mad_hi(a, b, c); }-static uint16_t futrts_umad_hi16(uint16_t a, uint16_t b, uint16_t c) { return mad_hi(a, b, c); }-static uint32_t futrts_umad_hi32(uint32_t a, uint32_t b, uint32_t c) { return mad_hi(a, b, c); }-static uint64_t futrts_umad_hi64(uint64_t a, uint64_t b, uint64_t c) { return mad_hi(a, b, c); }-static  uint8_t futrts_smad_hi8( int8_t a,  int8_t b,   int8_t c) { return mad_hi(a, b, c); }-static uint16_t futrts_smad_hi16(int16_t a, int16_t b, int16_t c) { return mad_hi(a, b, c); }-static uint32_t futrts_smad_hi32(int32_t a, int32_t b, int32_t c) { return mad_hi(a, b, c); }-static uint64_t futrts_smad_hi64(int64_t a, int64_t b, int64_t c) { return mad_hi(a, b, c); }-#else // Not OpenCL--static  uint8_t futrts_umad_hi8( uint8_t a,  uint8_t b,  uint8_t c) { return futrts_umul_hi8(a, b) + c; }-static uint16_t futrts_umad_hi16(uint16_t a, uint16_t b, uint16_t c) { return futrts_umul_hi16(a, b) + c; }-static uint32_t futrts_umad_hi32(uint32_t a, uint32_t b, uint32_t c) { return futrts_umul_hi32(a, b) + c; }-static uint64_t futrts_umad_hi64(uint64_t a, uint64_t b, uint64_t c) { return futrts_umul_hi64(a, b) + c; }-static  uint8_t futrts_smad_hi8 ( int8_t a,  int8_t b,  int8_t c) { return futrts_smul_hi8(a, b) + c; }-static uint16_t futrts_smad_hi16(int16_t a, int16_t b, int16_t c) { return futrts_smul_hi16(a, b) + c; }-static uint32_t futrts_smad_hi32(int32_t a, int32_t b, int32_t c) { return futrts_smul_hi32(a, b) + c; }-static uint64_t futrts_smad_hi64(int64_t a, int64_t b, int64_t c) { return futrts_smul_hi64(a, b) + c; }-#endif--#if defined(__OPENCL_VERSION__)-static int32_t futrts_clzz8(int8_t x) {-  return clz(x);-}--static int32_t futrts_clzz16(int16_t x) {-  return clz(x);-}--static int32_t futrts_clzz32(int32_t x) {-  return clz(x);-}--static int32_t futrts_clzz64(int64_t x) {-  return clz(x);-}--#elif defined(__CUDA_ARCH__)--static int32_t futrts_clzz8(int8_t x) {-  return __clz(zext_i8_i32(x)) - 24;-}--static int32_t futrts_clzz16(int16_t x) {-  return __clz(zext_i16_i32(x)) - 16;-}--static int32_t futrts_clzz32(int32_t x) {-  return __clz(x);-}--static int32_t futrts_clzz64(int64_t x) {-  return __clzll(x);-}--#elif ISPC--static int32_t futrts_clzz8(int8_t x) {-  return count_leading_zeros((int32_t)(uint8_t)x)-24;-}--static int32_t futrts_clzz16(int16_t x) {-  return count_leading_zeros((int32_t)(uint16_t)x)-16;-}--static int32_t futrts_clzz32(int32_t x) {-  return count_leading_zeros(x);-}--static int32_t futrts_clzz64(int64_t x) {-  return count_leading_zeros(x);-}--#else // Not OpenCL, ISPC or CUDA, but plain C.--static int32_t futrts_clzz8(int8_t x) {-  return x == 0 ? 8 : __builtin_clz((uint32_t)zext_i8_i32(x)) - 24;-}--static int32_t futrts_clzz16(int16_t x) {-  return x == 0 ? 16 : __builtin_clz((uint32_t)zext_i16_i32(x)) - 16;-}--static int32_t futrts_clzz32(int32_t x) {-  return x == 0 ? 32 : __builtin_clz((uint32_t)x);-}--static int32_t futrts_clzz64(int64_t x) {-  return x == 0 ? 64 : __builtin_clzll((uint64_t)x);-}-#endif--#if defined(__OPENCL_VERSION__)-static int32_t futrts_ctzz8(int8_t x) {-  int i = 0;-  for (; i < 8 && (x & 1) == 0; i++, x >>= 1)-    ;-  return i;-}--static int32_t futrts_ctzz16(int16_t x) {-  int i = 0;-  for (; i < 16 && (x & 1) == 0; i++, x >>= 1)-    ;-  return i;-}--static int32_t futrts_ctzz32(int32_t x) {-  int i = 0;-  for (; i < 32 && (x & 1) == 0; i++, x >>= 1)-    ;-  return i;-}--static int32_t futrts_ctzz64(int64_t x) {-  int i = 0;-  for (; i < 64 && (x & 1) == 0; i++, x >>= 1)-    ;-  return i;-}--#elif defined(__CUDA_ARCH__)--static int32_t futrts_ctzz8(int8_t x) {-  int y = __ffs(x);-  return y == 0 ? 8 : y - 1;-}--static int32_t futrts_ctzz16(int16_t x) {-  int y = __ffs(x);-  return y == 0 ? 16 : y - 1;-}--static int32_t futrts_ctzz32(int32_t x) {-  int y = __ffs(x);-  return y == 0 ? 32 : y - 1;-}--static int32_t futrts_ctzz64(int64_t x) {-  int y = __ffsll(x);-  return y == 0 ? 64 : y - 1;-}--#elif ISPC--static int32_t futrts_ctzz8(int8_t x) {-  return x == 0 ? 8 : count_trailing_zeros((int32_t)x);-}--static int32_t futrts_ctzz16(int16_t x) {-  return x == 0 ? 16 : count_trailing_zeros((int32_t)x);-}--static int32_t futrts_ctzz32(int32_t x) {-  return count_trailing_zeros(x);-}--static int32_t futrts_ctzz64(int64_t x) {-  return count_trailing_zeros(x);-}--#else // Not OpenCL or CUDA, but plain C.--static int32_t futrts_ctzz8(int8_t x) {-  return x == 0 ? 8 : __builtin_ctz((uint32_t)x);-}--static int32_t futrts_ctzz16(int16_t x) {-  return x == 0 ? 16 : __builtin_ctz((uint32_t)x);-}--static int32_t futrts_ctzz32(int32_t x) {-  return x == 0 ? 32 : __builtin_ctz((uint32_t)x);-}--static int32_t futrts_ctzz64(int64_t x) {-  return x == 0 ? 64 : __builtin_ctzll((uint64_t)x);-}-#endif--static inline float fdiv32(float x, float y) {-  return x / y;-}--static inline float fadd32(float x, float y) {-  return x + y;-}--static inline float fsub32(float x, float y) {-  return x - y;-}--static inline float fmul32(float x, float y) {-  return x * y;-}--static inline bool cmplt32(float x, float y) {-  return x < y;-}--static inline bool cmple32(float x, float y) {-  return x <= y;-}--static inline float sitofp_i8_f32(int8_t x) {-  return (float) x;-}--static inline float sitofp_i16_f32(int16_t x) {-  return (float) x;-}--static inline float sitofp_i32_f32(int32_t x) {-  return (float) x;-}--static inline float sitofp_i64_f32(int64_t x) {-  return (float) x;-}--static inline float uitofp_i8_f32(uint8_t x) {-  return (float) x;-}--static inline float uitofp_i16_f32(uint16_t x) {-  return (float) x;-}--static inline float uitofp_i32_f32(uint32_t x) {-  return (float) x;-}--static inline float uitofp_i64_f32(uint64_t x) {-  return (float) x;-}--#ifdef __OPENCL_VERSION__-static inline float fabs32(float x) {-  return fabs(x);-}--static inline float fmax32(float x, float y) {-  return fmax(x, y);-}--static inline float fmin32(float x, float y) {-  return fmin(x, y);-}--static inline float fpow32(float x, float y) {-  return pow(x, y);-}--#elif ISPC--static inline float fabs32(float x) {-  return abs(x);-}--static inline float fmax32(float x, float y) {-  return isnan(x) ? y : isnan(y) ? x : max(x, y);-}--static inline float fmin32(float x, float y) {-  return isnan(x) ? y : isnan(y) ? x : min(x, y);-}--static inline float fpow32(float a, float b) {-  float ret;-  foreach_active (i) {-      uniform float r = __stdlib_powf(extract(a, i), extract(b, i));-      ret = insert(ret, i, r);-  }-  return ret;-}--#else // Not OpenCL, but CUDA or plain C.--static inline float fabs32(float x) {-  return fabsf(x);-}--static inline float fmax32(float x, float y) {-  return fmaxf(x, y);-}--static inline float fmin32(float x, float y) {-  return fminf(x, y);-}--static inline float fpow32(float x, float y) {-  return powf(x, y);-}-#endif--static inline bool futrts_isnan32(float x) {-  return isnan(x);-}--#if ISPC--static inline bool futrts_isinf32(float x) {-  return !isnan(x) && isnan(x - x);-}--static inline bool futrts_isfinite32(float x) {-  return !isnan(x) && !futrts_isinf32(x);-}--#else--static inline bool futrts_isinf32(float x) {-  return isinf(x);-}--#endif--static inline int8_t fptosi_f32_i8(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (int8_t) x;-  }-}--static inline int16_t fptosi_f32_i16(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (int16_t) x;-  }-}--static inline int32_t fptosi_f32_i32(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (int32_t) x;-  }-}--static inline int64_t fptosi_f32_i64(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (int64_t) x;-  };-}--static inline uint8_t fptoui_f32_i8(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (uint8_t) (int8_t) x;-  }-}--static inline uint16_t fptoui_f32_i16(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (uint16_t) (int16_t) x;-  }-}--static inline uint32_t fptoui_f32_i32(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (uint32_t) (int32_t) x;-  }-}--static inline uint64_t fptoui_f32_i64(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (uint64_t) (int64_t) x;-  }-}--static inline bool ftob_f32_bool(float x) {-  return x != 0;-}--static inline float btof_bool_f32(bool x) {-  return x ? 1 : 0;-}--#ifdef __OPENCL_VERSION__-static inline float futrts_log32(float x) {-  return log(x);-}--static inline float futrts_log2_32(float x) {-  return log2(x);-}--static inline float futrts_log10_32(float x) {-  return log10(x);-}--static inline float futrts_log1p_32(float x) {-  return log1p(x);-}--static inline float futrts_sqrt32(float x) {-  return sqrt(x);-}--static inline float futrts_cbrt32(float x) {-  return cbrt(x);-}--static inline float futrts_exp32(float x) {-  return exp(x);-}--static inline float futrts_cos32(float x) {-  return cos(x);-}--static inline float futrts_sin32(float x) {-  return sin(x);-}--static inline float futrts_tan32(float x) {-  return tan(x);-}--static inline float futrts_acos32(float x) {-  return acos(x);-}--static inline float futrts_asin32(float x) {-  return asin(x);-}--static inline float futrts_atan32(float x) {-  return atan(x);-}--static inline float futrts_cosh32(float x) {-  return cosh(x);-}--static inline float futrts_sinh32(float x) {-  return sinh(x);-}--static inline float futrts_tanh32(float x) {-  return tanh(x);-}--static inline float futrts_acosh32(float x) {-  return acosh(x);-}--static inline float futrts_asinh32(float x) {-  return asinh(x);-}--static inline float futrts_atanh32(float x) {-  return atanh(x);-}--static inline float futrts_atan2_32(float x, float y) {-  return atan2(x, y);-}--static inline float futrts_hypot32(float x, float y) {-  return hypot(x, y);-}--static inline float futrts_gamma32(float x) {-  return tgamma(x);-}--static inline float futrts_lgamma32(float x) {-  return lgamma(x);-}--static inline float futrts_erf32(float x) {-  return erf(x);-}--static inline float futrts_erfc32(float x) {-  return erfc(x);-}--static inline float fmod32(float x, float y) {-  return fmod(x, y);-}--static inline float futrts_round32(float x) {-  return rint(x);-}--static inline float futrts_floor32(float x) {-  return floor(x);-}--static inline float futrts_ceil32(float x) {-  return ceil(x);-}--static inline float futrts_nextafter32(float x, float y) {-  return nextafter(x, y);-}--static inline float futrts_lerp32(float v0, float v1, float t) {-  return mix(v0, v1, t);-}--static inline float futrts_mad32(float a, float b, float c) {-  return mad(a, b, c);-}--static inline float futrts_fma32(float a, float b, float c) {-  return fma(a, b, c);-}--#elif ISPC--static inline float futrts_log32(float x) {-  return futrts_isfinite32(x) || (futrts_isinf32(x) && x < 0)? log(x) : x;-}--static inline float futrts_log2_32(float x) {-  return futrts_log32(x) / log(2.0f);-}--static inline float futrts_log10_32(float x) {-  return futrts_log32(x) / log(10.0f);-}--static inline float futrts_log1p_32(float x) {-  if(x == -1.0f || (futrts_isinf32(x) && x > 0.0f)) return x / 0.0f;-  float y = 1.0f + x;-  float z = y - 1.0f;-  return log(y) - (z-x)/y;-}--static inline float futrts_sqrt32(float x) {-  return sqrt(x);-}--extern "C" unmasked uniform float cbrtf(uniform float);-static inline float futrts_cbrt32(float x) {-  float res;-  foreach_active (i) {-    uniform float r = cbrtf(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--static inline float futrts_exp32(float x) {-  return exp(x);-}--static inline float futrts_cos32(float x) {-  return cos(x);-}--static inline float futrts_sin32(float x) {-  return sin(x);-}--static inline float futrts_tan32(float x) {-  return tan(x);-}--static inline float futrts_acos32(float x) {-  return acos(x);-}--static inline float futrts_asin32(float x) {-  return asin(x);-}--static inline float futrts_atan32(float x) {-  return atan(x);-}--static inline float futrts_cosh32(float x) {-  return (exp(x)+exp(-x)) / 2.0f;-}--static inline float futrts_sinh32(float x) {-  return (exp(x)-exp(-x)) / 2.0f;-}--static inline float futrts_tanh32(float x) {-  return futrts_sinh32(x)/futrts_cosh32(x);-}--static inline float futrts_acosh32(float x) {-  float f = x+sqrt(x*x-1);-  if(futrts_isfinite32(f)) return log(f);-  return f;-}--static inline float futrts_asinh32(float x) {-  float f = x+sqrt(x*x+1);-  if(futrts_isfinite32(f)) return log(f);-  return f;--}--static inline float futrts_atanh32(float x) {-  float f = (1+x)/(1-x);-  if(futrts_isfinite32(f)) return log(f)/2.0f;-  return f;--}--static inline float futrts_atan2_32(float x, float y) {-  return (x == 0.0f && y == 0.0f) ? 0.0f : atan2(x, y);-}--static inline float futrts_hypot32(float x, float y) {-  if (futrts_isfinite32(x) && futrts_isfinite32(y)) {-    x = abs(x);-    y = abs(y);-    float a;-    float b;-    if (x >= y){-        a = x;-        b = y;-    } else {-        a = y;-        b = x;-    }-    if(b == 0){-      return a;-    }--    int e;-    float an;-    float bn;-    an = frexp (a, &e);-    bn = ldexp (b, - e);-    float cn;-    cn = sqrt (an * an + bn * bn);-    return ldexp (cn, e);-  } else {-    if (futrts_isinf32(x) || futrts_isinf32(y)) return INFINITY;-    else return x + y;-  }--}--extern "C" unmasked uniform float tgammaf(uniform float x);-static inline float futrts_gamma32(float x) {-  float res;-  foreach_active (i) {-    uniform float r = tgammaf(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--extern "C" unmasked uniform float lgammaf(uniform float x);-static inline float futrts_lgamma32(float x) {-  float res;-  foreach_active (i) {-    uniform float r = lgammaf(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--extern "C" unmasked uniform float erff(uniform float x);-static inline float futrts_erf32(float x) {-  float res;-  foreach_active (i) {-    uniform float r = erff(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--extern "C" unmasked uniform float erfcf(uniform float x);-static inline float futrts_erfc32(float x) {-  float res;-  foreach_active (i) {-    uniform float r = erfcf(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--static inline float fmod32(float x, float y) {-  return x - y * trunc(x/y);-}--static inline float futrts_round32(float x) {-  return round(x);-}--static inline float futrts_floor32(float x) {-  return floor(x);-}--static inline float futrts_ceil32(float x) {-  return ceil(x);-}--extern "C" unmasked uniform float nextafterf(uniform float x, uniform float y);-static inline float futrts_nextafter32(float x, float y) {-  float res;-  foreach_active (i) {-    uniform float r = nextafterf(extract(x, i), extract(y, i));-    res = insert(res, i, r);-  }-  return res;-}--static inline float futrts_lerp32(float v0, float v1, float t) {-  return v0 + (v1 - v0) * t;-}--static inline float futrts_mad32(float a, float b, float c) {-  return a * b + c;-}--static inline float futrts_fma32(float a, float b, float c) {-  return a * b + c;-}--#else // Not OpenCL or ISPC, but CUDA or plain C.--static inline float futrts_log32(float x) {-  return logf(x);-}--static inline float futrts_log2_32(float x) {-  return log2f(x);-}--static inline float futrts_log10_32(float x) {-  return log10f(x);-}--static inline float futrts_log1p_32(float x) {-  return log1pf(x);-}--static inline float futrts_sqrt32(float x) {-  return sqrtf(x);-}--static inline float futrts_cbrt32(float x) {-  return cbrtf(x);-}--static inline float futrts_exp32(float x) {-  return expf(x);-}--static inline float futrts_cos32(float x) {-  return cosf(x);-}--static inline float futrts_sin32(float x) {-  return sinf(x);-}--static inline float futrts_tan32(float x) {-  return tanf(x);-}--static inline float futrts_acos32(float x) {-  return acosf(x);-}--static inline float futrts_asin32(float x) {-  return asinf(x);-}--static inline float futrts_atan32(float x) {-  return atanf(x);-}--static inline float futrts_cosh32(float x) {-  return coshf(x);-}--static inline float futrts_sinh32(float x) {-  return sinhf(x);-}--static inline float futrts_tanh32(float x) {-  return tanhf(x);-}--static inline float futrts_acosh32(float x) {-  return acoshf(x);-}--static inline float futrts_asinh32(float x) {-  return asinhf(x);-}--static inline float futrts_atanh32(float x) {-  return atanhf(x);-}--static inline float futrts_atan2_32(float x, float y) {-  return atan2f(x, y);-}--static inline float futrts_hypot32(float x, float y) {-  return hypotf(x, y);-}--static inline float futrts_gamma32(float x) {-  return tgammaf(x);-}--static inline float futrts_lgamma32(float x) {-  return lgammaf(x);-}--static inline float futrts_erf32(float x) {-  return erff(x);-}--static inline float futrts_erfc32(float x) {-  return erfcf(x);-}--static inline float fmod32(float x, float y) {-  return fmodf(x, y);-}--static inline float futrts_round32(float x) {-  return rintf(x);-}--static inline float futrts_floor32(float x) {-  return floorf(x);-}--static inline float futrts_ceil32(float x) {-  return ceilf(x);-}--static inline float futrts_nextafter32(float x, float y) {-  return nextafterf(x, y);-}--static inline float futrts_lerp32(float v0, float v1, float t) {-  return v0 + (v1 - v0) * t;-}--static inline float futrts_mad32(float a, float b, float c) {-  return a * b + c;-}--static inline float futrts_fma32(float a, float b, float c) {-  return fmaf(a, b, c);-}-#endif--#if ISPC-static inline int32_t futrts_to_bits32(float x) {-  return intbits(x);-}--static inline float futrts_from_bits32(int32_t x) {-  return floatbits(x);-}-#else-static inline int32_t futrts_to_bits32(float x) {-  union {-    float f;-    int32_t t;-  } p;--  p.f = x;-  return p.t;-}--static inline float futrts_from_bits32(int32_t x) {-  union {-    int32_t f;-    float t;-  } p;--  p.f = x;-  return p.t;-}-#endif--static inline float fsignum32(float x) {-  return futrts_isnan32(x) ? x : (x > 0 ? 1 : 0) - (x < 0 ? 1 : 0);-}--#ifdef FUTHARK_F64_ENABLED--#if ISPC-static inline bool futrts_isinf64(float x) {-  return !isnan(x) && isnan(x - x);-}--static inline bool futrts_isfinite64(float x) {-  return !isnan(x) && !futrts_isinf64(x);-}--static inline double fdiv64(double x, double y) {-  return x / y;-}--static inline double fadd64(double x, double y) {-  return x + y;-}--static inline double fsub64(double x, double y) {-  return x - y;-}--static inline double fmul64(double x, double y) {-  return x * y;-}--static inline bool cmplt64(double x, double y) {-  return x < y;-}--static inline bool cmple64(double x, double y) {-  return x <= y;-}--static inline double sitofp_i8_f64(int8_t x) {-  return (double) x;-}--static inline double sitofp_i16_f64(int16_t x) {-  return (double) x;-}--static inline double sitofp_i32_f64(int32_t x) {-  return (double) x;-}--static inline double sitofp_i64_f64(int64_t x) {-  return (double) x;-}--static inline double uitofp_i8_f64(uint8_t x) {-  return (double) x;-}--static inline double uitofp_i16_f64(uint16_t x) {-  return (double) x;-}--static inline double uitofp_i32_f64(uint32_t x) {-  return (double) x;-}--static inline double uitofp_i64_f64(uint64_t x) {-  return (double) x;-}--static inline double fabs64(double x) {-  return abs(x);-}--static inline double fmax64(double x, double y) {-  return isnan(x) ? y : isnan(y) ? x : max(x, y);-}--static inline double fmin64(double x, double y) {-  return isnan(x) ? y : isnan(y) ? x : min(x, y);-}--static inline double fpow64(double a, double b) {-  float ret;-  foreach_active (i) {-      uniform float r = __stdlib_powf(extract(a, i), extract(b, i));-      ret = insert(ret, i, r);-  }-  return ret;-}--static inline double futrts_log64(double x) {-  return futrts_isfinite64(x) || (futrts_isinf64(x) && x < 0)? log(x) : x;-}--static inline double futrts_log2_64(double x) {-  return futrts_log64(x)/log(2.0d);-}--static inline double futrts_log10_64(double x) {-  return futrts_log64(x)/log(10.0d);-}--static inline double futrts_log1p_64(double x) {-  if(x == -1.0d || (futrts_isinf64(x) && x > 0.0d)) return x / 0.0d;-  double y = 1.0d + x;-  double z = y - 1.0d;-  return log(y) - (z-x)/y;-}--static inline double futrts_sqrt64(double x) {-  return sqrt(x);-}--extern "C" unmasked uniform double cbrt(uniform double);-static inline double futrts_cbrt64(double x) {-  double res;-  foreach_active (i) {-    uniform double r = cbrtf(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--static inline double futrts_exp64(double x) {-  return exp(x);-}--static inline double futrts_cos64(double x) {-  return cos(x);-}--static inline double futrts_sin64(double x) {-  return sin(x);-}--static inline double futrts_tan64(double x) {-  return tan(x);-}--static inline double futrts_acos64(double x) {-  return acos(x);-}--static inline double futrts_asin64(double x) {-  return asin(x);-}--static inline double futrts_atan64(double x) {-  return atan(x);-}--static inline double futrts_cosh64(double x) {-  return (exp(x)+exp(-x)) / 2.0d;-}--static inline double futrts_sinh64(double x) {-  return (exp(x)-exp(-x)) / 2.0d;-}--static inline double futrts_tanh64(double x) {-  return futrts_sinh64(x)/futrts_cosh64(x);-}--static inline double futrts_acosh64(double x) {-  double f = x+sqrt(x*x-1.0d);-  if(futrts_isfinite64(f)) return log(f);-  return f;-}--static inline double futrts_asinh64(double x) {-  double f = x+sqrt(x*x+1.0d);-  if(futrts_isfinite64(f)) return log(f);-  return f;-}--static inline double futrts_atanh64(double x) {-  double f = (1.0d+x)/(1.0d-x);-  if(futrts_isfinite64(f)) return log(f)/2.0d;-  return f;--}--static inline double futrts_atan2_64(double x, double y) {-  return atan2(x, y);-}--extern "C" unmasked uniform double hypot(uniform double x, uniform double y);-static inline double futrts_hypot64(double x, double y) {-  double res;-  foreach_active (i) {-    uniform double r = hypot(extract(x, i), extract(y, i));-    res = insert(res, i, r);-  }-  return res;-}--extern "C" unmasked uniform double tgamma(uniform double x);-static inline double futrts_gamma64(double x) {-  double res;-  foreach_active (i) {-    uniform double r = tgamma(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--extern "C" unmasked uniform double lgamma(uniform double x);-static inline double futrts_lgamma64(double x) {-  double res;-  foreach_active (i) {-    uniform double r = lgamma(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--extern "C" unmasked uniform double erf(uniform double x);-static inline double futrts_erf64(double x) {-  double res;-  foreach_active (i) {-    uniform double r = erf(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--extern "C" unmasked uniform double erfc(uniform double x);-static inline double futrts_erfc64(double x) {-  double res;-  foreach_active (i) {-    uniform double r = erfc(extract(x, i));-    res = insert(res, i, r);-  }-  return res;-}--static inline double futrts_fma64(double a, double b, double c) {-  return a * b + c;-}--static inline double futrts_round64(double x) {-  return round(x);-}--static inline double futrts_ceil64(double x) {-  return ceil(x);-}--extern "C" unmasked uniform double nextafter(uniform float x, uniform double y);-static inline float futrts_nextafter64(double x, double y) {-  double res;-  foreach_active (i) {-    uniform double r = nextafter(extract(x, i), extract(y, i));-    res = insert(res, i, r);-  }-  return res;-}--static inline double futrts_floor64(double x) {-  return floor(x);-}--static inline bool futrts_isnan64(double x) {-  return isnan(x);-}--static inline int8_t fptosi_f64_i8(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int8_t) x;-  }-}--static inline int16_t fptosi_f64_i16(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int16_t) x;-  }-}--static inline int32_t fptosi_f64_i32(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int32_t) x;-  }-}--static inline int64_t fptosi_f64_i64(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int64_t) x;-  }-}--static inline uint8_t fptoui_f64_i8(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint8_t) (int8_t) x;-  }-}--static inline uint16_t fptoui_f64_i16(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint16_t) (int16_t) x;-  }-}--static inline uint32_t fptoui_f64_i32(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint32_t) (int32_t) x;-  }-}--static inline uint64_t fptoui_f64_i64(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint64_t) (int64_t) x;-  }-}--static inline bool ftob_f64_bool(double x) {-  return x != 0.0;-}--static inline double btof_bool_f64(bool x) {-  return x ? 1.0 : 0.0;-}--static inline int64_t futrts_to_bits64(double x) {-  int64_t res;-  foreach_active (i) {-    uniform double tmp = extract(x, i);-    uniform int64_t r = *((uniform int64_t* uniform)&tmp);-    res = insert(res, i, r);-  }-  return res;-}--static inline double futrts_from_bits64(int64_t x) {-  double res;-  foreach_active (i) {-    uniform int64_t tmp = extract(x, i);-    uniform double r = *((uniform double* uniform)&tmp);-    res = insert(res, i, r);-  }-  return res;-}--static inline double fmod64(double x, double y) {-  return x - y * trunc(x/y);-}--static inline double fsignum64(double x) {-  return futrts_isnan64(x) ? x : (x > 0 ? 1.0d : 0.0d) - (x < 0 ? 1.0d : 0.0d);-}--static inline double futrts_lerp64(double v0, double v1, double t) {-  return v0 + (v1 - v0) * t;-}--static inline double futrts_mad64(double a, double b, double c) {-  return a * b + c;-}--static inline float fpconv_f32_f32(float x) {-  return (float) x;-}--static inline double fpconv_f32_f64(float x) {-  return (double) x;-}--static inline float fpconv_f64_f32(double x) {-  return (float) x;-}--static inline double fpconv_f64_f64(double x) {-  return (double) x;-}--#else--static inline double fdiv64(double x, double y) {-  return x / y;-}--static inline double fadd64(double x, double y) {-  return x + y;-}--static inline double fsub64(double x, double y) {-  return x - y;-}--static inline double fmul64(double x, double y) {-  return x * y;-}--static inline bool cmplt64(double x, double y) {-  return x < y;-}--static inline bool cmple64(double x, double y) {-  return x <= y;-}--static inline double sitofp_i8_f64(int8_t x) {-  return (double) x;-}--static inline double sitofp_i16_f64(int16_t x) {-  return (double) x;-}--static inline double sitofp_i32_f64(int32_t x) {-  return (double) x;-}--static inline double sitofp_i64_f64(int64_t x) {-  return (double) x;-}--static inline double uitofp_i8_f64(uint8_t x) {-  return (double) x;-}--static inline double uitofp_i16_f64(uint16_t x) {-  return (double) x;-}--static inline double uitofp_i32_f64(uint32_t x) {-  return (double) x;-}--static inline double uitofp_i64_f64(uint64_t x) {-  return (double) x;-}--static inline double fabs64(double x) {-  return fabs(x);-}--static inline double fmax64(double x, double y) {-  return fmax(x, y);-}--static inline double fmin64(double x, double y) {-  return fmin(x, y);-}--static inline double fpow64(double x, double y) {-  return pow(x, y);-}--static inline double futrts_log64(double x) {-  return log(x);-}--static inline double futrts_log2_64(double x) {-  return log2(x);-}--static inline double futrts_log10_64(double x) {-  return log10(x);-}--static inline double futrts_log1p_64(double x) {-  return log1p(x);-}--static inline double futrts_sqrt64(double x) {-  return sqrt(x);-}--static inline double futrts_cbrt64(double x) {-  return cbrt(x);-}--static inline double futrts_exp64(double x) {-  return exp(x);-}--static inline double futrts_cos64(double x) {-  return cos(x);-}--static inline double futrts_sin64(double x) {-  return sin(x);-}--static inline double futrts_tan64(double x) {-  return tan(x);-}--static inline double futrts_acos64(double x) {-  return acos(x);-}--static inline double futrts_asin64(double x) {-  return asin(x);-}--static inline double futrts_atan64(double x) {-  return atan(x);-}--static inline double futrts_cosh64(double x) {-  return cosh(x);-}--static inline double futrts_sinh64(double x) {-  return sinh(x);-}--static inline double futrts_tanh64(double x) {-  return tanh(x);-}--static inline double futrts_acosh64(double x) {-  return acosh(x);-}--static inline double futrts_asinh64(double x) {-  return asinh(x);-}--static inline double futrts_atanh64(double x) {-  return atanh(x);-}--static inline double futrts_atan2_64(double x, double y) {-  return atan2(x, y);-}--static inline double futrts_hypot64(double x, double y) {-  return hypot(x, y);-}--static inline double futrts_gamma64(double x) {-  return tgamma(x);-}--static inline double futrts_lgamma64(double x) {-  return lgamma(x);-}--static inline double futrts_erf64(double x) {-  return erf(x);-}--static inline double futrts_erfc64(double x) {-  return erfc(x);-}--static inline double futrts_fma64(double a, double b, double c) {-  return fma(a, b, c);-}--static inline double futrts_round64(double x) {-  return rint(x);-}--static inline double futrts_ceil64(double x) {-  return ceil(x);-}--static inline float futrts_nextafter64(float x, float y) {-  return nextafter(x, y);-}--static inline double futrts_floor64(double x) {-  return floor(x);-}--static inline bool futrts_isnan64(double x) {-  return isnan(x);-}--static inline bool futrts_isinf64(double x) {-  return isinf(x);-}--static inline int8_t fptosi_f64_i8(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int8_t) x;-  }-}--static inline int16_t fptosi_f64_i16(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int16_t) x;-  }-}--static inline int32_t fptosi_f64_i32(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int32_t) x;-  }-}--static inline int64_t fptosi_f64_i64(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int64_t) x;-  }-}--static inline uint8_t fptoui_f64_i8(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint8_t) (int8_t) x;-  }-}--static inline uint16_t fptoui_f64_i16(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint16_t) (int16_t) x;-  }-}--static inline uint32_t fptoui_f64_i32(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint32_t) (int32_t) x;-  }-}--static inline uint64_t fptoui_f64_i64(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint64_t) (int64_t) x;-  }-}--static inline bool ftob_f64_bool(double x) {-  return x != 0;-}--static inline double btof_bool_f64(bool x) {-  return x ? 1 : 0;-}--static inline int64_t futrts_to_bits64(double x) {-  union {-    double f;-    int64_t t;-  } p;--  p.f = x;-  return p.t;-}--static inline double futrts_from_bits64(int64_t x) {-  union {-    int64_t f;-    double t;-  } p;--  p.f = x;-  return p.t;-}--static inline double fmod64(double x, double y) {-  return fmod(x, y);-}--static inline double fsignum64(double x) {-  return futrts_isnan64(x) ? x : (x > 0) - (x < 0);-}--static inline double futrts_lerp64(double v0, double v1, double t) {-#ifdef __OPENCL_VERSION__-  return mix(v0, v1, t);-#else-  return v0 + (v1 - v0) * t;-#endif-}--static inline double futrts_mad64(double a, double b, double c) {-#ifdef __OPENCL_VERSION__-  return mad(a, b, c);-#else-  return a * b + c;-#endif-}--static inline float fpconv_f32_f32(float x) {-  return (float) x;-}--static inline double fpconv_f32_f64(float x) {-  return (double) x;-}--static inline float fpconv_f64_f32(double x) {-  return (float) x;-}--static inline double fpconv_f64_f64(double x) {+SCALAR_FUN_ATTR uint8_t add8(uint8_t x, uint8_t y) {+  return x + y;+}++SCALAR_FUN_ATTR uint16_t add16(uint16_t x, uint16_t y) {+  return x + y;+}++SCALAR_FUN_ATTR uint32_t add32(uint32_t x, uint32_t y) {+  return x + y;+}++SCALAR_FUN_ATTR uint64_t add64(uint64_t x, uint64_t y) {+  return x + y;+}++SCALAR_FUN_ATTR uint8_t sub8(uint8_t x, uint8_t y) {+  return x - y;+}++SCALAR_FUN_ATTR uint16_t sub16(uint16_t x, uint16_t y) {+  return x - y;+}++SCALAR_FUN_ATTR uint32_t sub32(uint32_t x, uint32_t y) {+  return x - y;+}++SCALAR_FUN_ATTR uint64_t sub64(uint64_t x, uint64_t y) {+  return x - y;+}++SCALAR_FUN_ATTR uint8_t mul8(uint8_t x, uint8_t y) {+  return x * y;+}++SCALAR_FUN_ATTR uint16_t mul16(uint16_t x, uint16_t y) {+  return x * y;+}++SCALAR_FUN_ATTR uint32_t mul32(uint32_t x, uint32_t y) {+  return x * y;+}++SCALAR_FUN_ATTR uint64_t mul64(uint64_t x, uint64_t y) {+  return x * y;+}++#if ISPC++SCALAR_FUN_ATTR uint8_t udiv8(uint8_t x, uint8_t y) {+  // This strange pattern is used to prevent the ISPC compiler from+  // causing SIGFPEs and bogus results on divisions where inactive lanes+  // have 0-valued divisors. It ensures that any inactive lane instead+  // has a divisor of 1. https://github.com/ispc/ispc/issues/2292+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x / ys;+}++SCALAR_FUN_ATTR uint16_t udiv16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x / ys;+}++SCALAR_FUN_ATTR uint32_t udiv32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+++  return x / ys;+}++SCALAR_FUN_ATTR uint64_t udiv64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+++  return x / ys;+}++SCALAR_FUN_ATTR uint8_t udiv_up8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+++  return (x + y - 1) / ys;+}++SCALAR_FUN_ATTR uint16_t udiv_up16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return (x + y - 1) / ys;+}++SCALAR_FUN_ATTR uint32_t udiv_up32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return (x + y - 1) / ys;+}++SCALAR_FUN_ATTR uint64_t udiv_up64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return (x + y - 1) / ys;+}++SCALAR_FUN_ATTR uint8_t umod8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x % ys;+}++SCALAR_FUN_ATTR uint16_t umod16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+++  return x % ys;+}++SCALAR_FUN_ATTR uint32_t umod32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x % ys;+}++SCALAR_FUN_ATTR uint64_t umod64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x % ys;+}++SCALAR_FUN_ATTR uint8_t udiv_safe8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x / ys;+}++SCALAR_FUN_ATTR uint16_t udiv_safe16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x / ys;+}++SCALAR_FUN_ATTR uint32_t udiv_safe32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x / ys;+}++SCALAR_FUN_ATTR uint64_t udiv_safe64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x / ys;+}++SCALAR_FUN_ATTR uint8_t udiv_up_safe8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : (x + y - 1) / ys;+}++SCALAR_FUN_ATTR uint16_t udiv_up_safe16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : (x + y - 1) / ys;+}++SCALAR_FUN_ATTR uint32_t udiv_up_safe32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : (x + y - 1) / ys;+}++SCALAR_FUN_ATTR uint64_t udiv_up_safe64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : (x + y - 1) / ys;+}++SCALAR_FUN_ATTR uint8_t umod_safe8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x % ys;+}++SCALAR_FUN_ATTR uint16_t umod_safe16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x % ys;+}++SCALAR_FUN_ATTR uint32_t umod_safe32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x % ys;+}++SCALAR_FUN_ATTR uint64_t umod_safe64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x % ys;+}++SCALAR_FUN_ATTR int8_t sdiv8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  int8_t q = x / ys;+  int8_t r = x % ys;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++SCALAR_FUN_ATTR int16_t sdiv16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  int16_t q = x / ys;+  int16_t r = x % ys;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++SCALAR_FUN_ATTR int32_t sdiv32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  int32_t q = x / ys;+  int32_t r = x % ys;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++SCALAR_FUN_ATTR int64_t sdiv64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  int64_t q = x / ys;+  int64_t r = x % ys;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++SCALAR_FUN_ATTR int8_t sdiv_up8(int8_t x, int8_t y) {+  return sdiv8(x + y - 1, y);+}++SCALAR_FUN_ATTR int16_t sdiv_up16(int16_t x, int16_t y) {+  return sdiv16(x + y - 1, y);+}++SCALAR_FUN_ATTR int32_t sdiv_up32(int32_t x, int32_t y) {+  return sdiv32(x + y - 1, y);+}++SCALAR_FUN_ATTR int64_t sdiv_up64(int64_t x, int64_t y) {+  return sdiv64(x + y - 1, y);+}++SCALAR_FUN_ATTR int8_t smod8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  int8_t r = x % ys;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++SCALAR_FUN_ATTR int16_t smod16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  int16_t r = x % ys;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++SCALAR_FUN_ATTR int32_t smod32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  int32_t r = x % ys;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++SCALAR_FUN_ATTR int64_t smod64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  int64_t r = x % ys;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++SCALAR_FUN_ATTR int8_t sdiv_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : sdiv8(x, y);+}++SCALAR_FUN_ATTR int16_t sdiv_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : sdiv16(x, y);+}++SCALAR_FUN_ATTR int32_t sdiv_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : sdiv32(x, y);+}++SCALAR_FUN_ATTR int64_t sdiv_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : sdiv64(x, y);+}++SCALAR_FUN_ATTR int8_t sdiv_up_safe8(int8_t x, int8_t y) {+  return sdiv_safe8(x + y - 1, y);+}++SCALAR_FUN_ATTR int16_t sdiv_up_safe16(int16_t x, int16_t y) {+  return sdiv_safe16(x + y - 1, y);+}++SCALAR_FUN_ATTR int32_t sdiv_up_safe32(int32_t x, int32_t y) {+  return sdiv_safe32(x + y - 1, y);+}++SCALAR_FUN_ATTR int64_t sdiv_up_safe64(int64_t x, int64_t y) {+  return sdiv_safe64(x + y - 1, y);+}++SCALAR_FUN_ATTR int8_t smod_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : smod8(x, y);+}++SCALAR_FUN_ATTR int16_t smod_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : smod16(x, y);+}++SCALAR_FUN_ATTR int32_t smod_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : smod32(x, y);+}++SCALAR_FUN_ATTR int64_t smod_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : smod64(x, y);+}++SCALAR_FUN_ATTR int8_t squot8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x / ys;+}++SCALAR_FUN_ATTR int16_t squot16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x / ys;+}++SCALAR_FUN_ATTR int32_t squot32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x / ys;+}++SCALAR_FUN_ATTR int64_t squot64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x / ys;+}++SCALAR_FUN_ATTR int8_t srem8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x % ys;+}++SCALAR_FUN_ATTR int16_t srem16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x % ys;+}++SCALAR_FUN_ATTR int32_t srem32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x % ys;+}++SCALAR_FUN_ATTR int64_t srem64(int64_t x, int64_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x % ys;+}++SCALAR_FUN_ATTR int8_t squot_safe8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x / ys;+}++SCALAR_FUN_ATTR int16_t squot_safe16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x / ys;+}++SCALAR_FUN_ATTR int32_t squot_safe32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x / ys;+}++SCALAR_FUN_ATTR int64_t squot_safe64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x / ys;+}++SCALAR_FUN_ATTR int8_t srem_safe8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x % ys;+}++SCALAR_FUN_ATTR int16_t srem_safe16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x % ys;+}++SCALAR_FUN_ATTR int32_t srem_safe32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x % ys;+}++SCALAR_FUN_ATTR int64_t srem_safe64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return y == 0 ? 0 : x % ys;+}++#else++SCALAR_FUN_ATTR uint8_t udiv8(uint8_t x, uint8_t y) {+  return x / y;+}++SCALAR_FUN_ATTR uint16_t udiv16(uint16_t x, uint16_t y) {+  return x / y;+}++SCALAR_FUN_ATTR uint32_t udiv32(uint32_t x, uint32_t y) {+  return x / y;+}++SCALAR_FUN_ATTR uint64_t udiv64(uint64_t x, uint64_t y) {+  return x / y;+}++SCALAR_FUN_ATTR uint8_t udiv_up8(uint8_t x, uint8_t y) {+  return (x + y - 1) / y;+}++SCALAR_FUN_ATTR uint16_t udiv_up16(uint16_t x, uint16_t y) {+  return (x + y - 1) / y;+}++SCALAR_FUN_ATTR uint32_t udiv_up32(uint32_t x, uint32_t y) {+  return (x + y - 1) / y;+}++SCALAR_FUN_ATTR uint64_t udiv_up64(uint64_t x, uint64_t y) {+  return (x + y - 1) / y;+}++SCALAR_FUN_ATTR uint8_t umod8(uint8_t x, uint8_t y) {+  return x % y;+}++SCALAR_FUN_ATTR uint16_t umod16(uint16_t x, uint16_t y) {+  return x % y;+}++SCALAR_FUN_ATTR uint32_t umod32(uint32_t x, uint32_t y) {+  return x % y;+}++SCALAR_FUN_ATTR uint64_t umod64(uint64_t x, uint64_t y) {+  return x % y;+}++SCALAR_FUN_ATTR uint8_t udiv_safe8(uint8_t x, uint8_t y) {+  return y == 0 ? 0 : x / y;+}++SCALAR_FUN_ATTR uint16_t udiv_safe16(uint16_t x, uint16_t y) {+  return y == 0 ? 0 : x / y;+}++SCALAR_FUN_ATTR uint32_t udiv_safe32(uint32_t x, uint32_t y) {+  return y == 0 ? 0 : x / y;+}++SCALAR_FUN_ATTR uint64_t udiv_safe64(uint64_t x, uint64_t y) {+  return y == 0 ? 0 : x / y;+}++SCALAR_FUN_ATTR uint8_t udiv_up_safe8(uint8_t x, uint8_t y) {+  return y == 0 ? 0 : (x + y - 1) / y;+}++SCALAR_FUN_ATTR uint16_t udiv_up_safe16(uint16_t x, uint16_t y) {+  return y == 0 ? 0 : (x + y - 1) / y;+}++SCALAR_FUN_ATTR uint32_t udiv_up_safe32(uint32_t x, uint32_t y) {+  return y == 0 ? 0 : (x + y - 1) / y;+}++SCALAR_FUN_ATTR uint64_t udiv_up_safe64(uint64_t x, uint64_t y) {+  return y == 0 ? 0 : (x + y - 1) / y;+}++SCALAR_FUN_ATTR uint8_t umod_safe8(uint8_t x, uint8_t y) {+  return y == 0 ? 0 : x % y;+}++SCALAR_FUN_ATTR uint16_t umod_safe16(uint16_t x, uint16_t y) {+  return y == 0 ? 0 : x % y;+}++SCALAR_FUN_ATTR uint32_t umod_safe32(uint32_t x, uint32_t y) {+  return y == 0 ? 0 : x % y;+}++SCALAR_FUN_ATTR uint64_t umod_safe64(uint64_t x, uint64_t y) {+  return y == 0 ? 0 : x % y;+}++SCALAR_FUN_ATTR int8_t sdiv8(int8_t x, int8_t y) {+  int8_t q = x / y;+  int8_t r = x % y;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++SCALAR_FUN_ATTR int16_t sdiv16(int16_t x, int16_t y) {+  int16_t q = x / y;+  int16_t r = x % y;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++SCALAR_FUN_ATTR int32_t sdiv32(int32_t x, int32_t y) {+  int32_t q = x / y;+  int32_t r = x % y;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++SCALAR_FUN_ATTR int64_t sdiv64(int64_t x, int64_t y) {+  int64_t q = x / y;+  int64_t r = x % y;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++SCALAR_FUN_ATTR int8_t sdiv_up8(int8_t x, int8_t y) {+  return sdiv8(x + y - 1, y);+}++SCALAR_FUN_ATTR int16_t sdiv_up16(int16_t x, int16_t y) {+  return sdiv16(x + y - 1, y);+}++SCALAR_FUN_ATTR int32_t sdiv_up32(int32_t x, int32_t y) {+  return sdiv32(x + y - 1, y);+}++SCALAR_FUN_ATTR int64_t sdiv_up64(int64_t x, int64_t y) {+  return sdiv64(x + y - 1, y);+}++SCALAR_FUN_ATTR int8_t smod8(int8_t x, int8_t y) {+  int8_t r = x % y;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++SCALAR_FUN_ATTR int16_t smod16(int16_t x, int16_t y) {+  int16_t r = x % y;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++SCALAR_FUN_ATTR int32_t smod32(int32_t x, int32_t y) {+  int32_t r = x % y;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++SCALAR_FUN_ATTR int64_t smod64(int64_t x, int64_t y) {+  int64_t r = x % y;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++SCALAR_FUN_ATTR int8_t sdiv_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : sdiv8(x, y);+}++SCALAR_FUN_ATTR int16_t sdiv_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : sdiv16(x, y);+}++SCALAR_FUN_ATTR int32_t sdiv_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : sdiv32(x, y);+}++SCALAR_FUN_ATTR int64_t sdiv_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : sdiv64(x, y);+}++SCALAR_FUN_ATTR int8_t sdiv_up_safe8(int8_t x, int8_t y) {+  return sdiv_safe8(x + y - 1, y);+}++SCALAR_FUN_ATTR int16_t sdiv_up_safe16(int16_t x, int16_t y) {+  return sdiv_safe16(x + y - 1, y);+}++SCALAR_FUN_ATTR int32_t sdiv_up_safe32(int32_t x, int32_t y) {+  return sdiv_safe32(x + y - 1, y);+}++SCALAR_FUN_ATTR int64_t sdiv_up_safe64(int64_t x, int64_t y) {+  return sdiv_safe64(x + y - 1, y);+}++SCALAR_FUN_ATTR int8_t smod_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : smod8(x, y);+}++SCALAR_FUN_ATTR int16_t smod_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : smod16(x, y);+}++SCALAR_FUN_ATTR int32_t smod_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : smod32(x, y);+}++SCALAR_FUN_ATTR int64_t smod_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : smod64(x, y);+}++SCALAR_FUN_ATTR int8_t squot8(int8_t x, int8_t y) {+  return x / y;+}++SCALAR_FUN_ATTR int16_t squot16(int16_t x, int16_t y) {+  return x / y;+}++SCALAR_FUN_ATTR int32_t squot32(int32_t x, int32_t y) {+  return x / y;+}++SCALAR_FUN_ATTR int64_t squot64(int64_t x, int64_t y) {+  return x / y;+}++SCALAR_FUN_ATTR int8_t srem8(int8_t x, int8_t y) {+  return x % y;+}++SCALAR_FUN_ATTR int16_t srem16(int16_t x, int16_t y) {+  return x % y;+}++SCALAR_FUN_ATTR int32_t srem32(int32_t x, int32_t y) {+  return x % y;+}++SCALAR_FUN_ATTR int64_t srem64(int64_t x, int64_t y) {+  return x % y;+}++SCALAR_FUN_ATTR int8_t squot_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : x / y;+}++SCALAR_FUN_ATTR int16_t squot_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : x / y;+}++SCALAR_FUN_ATTR int32_t squot_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : x / y;+}++SCALAR_FUN_ATTR int64_t squot_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : x / y;+}++SCALAR_FUN_ATTR int8_t srem_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : x % y;+}++SCALAR_FUN_ATTR int16_t srem_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : x % y;+}++SCALAR_FUN_ATTR int32_t srem_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : x % y;+}++SCALAR_FUN_ATTR int64_t srem_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : x % y;+}++#endif++SCALAR_FUN_ATTR int8_t smin8(int8_t x, int8_t y) {+  return x < y ? x : y;+}++SCALAR_FUN_ATTR int16_t smin16(int16_t x, int16_t y) {+  return x < y ? x : y;+}++SCALAR_FUN_ATTR int32_t smin32(int32_t x, int32_t y) {+  return x < y ? x : y;+}++SCALAR_FUN_ATTR int64_t smin64(int64_t x, int64_t y) {+  return x < y ? x : y;+}++SCALAR_FUN_ATTR uint8_t umin8(uint8_t x, uint8_t y) {+  return x < y ? x : y;+}++SCALAR_FUN_ATTR uint16_t umin16(uint16_t x, uint16_t y) {+  return x < y ? x : y;+}++SCALAR_FUN_ATTR uint32_t umin32(uint32_t x, uint32_t y) {+  return x < y ? x : y;+}++SCALAR_FUN_ATTR uint64_t umin64(uint64_t x, uint64_t y) {+  return x < y ? x : y;+}++SCALAR_FUN_ATTR int8_t smax8(int8_t x, int8_t y) {+  return x < y ? y : x;+}++SCALAR_FUN_ATTR int16_t smax16(int16_t x, int16_t y) {+  return x < y ? y : x;+}++SCALAR_FUN_ATTR int32_t smax32(int32_t x, int32_t y) {+  return x < y ? y : x;+}++SCALAR_FUN_ATTR int64_t smax64(int64_t x, int64_t y) {+  return x < y ? y : x;+}++SCALAR_FUN_ATTR uint8_t umax8(uint8_t x, uint8_t y) {+  return x < y ? y : x;+}++SCALAR_FUN_ATTR uint16_t umax16(uint16_t x, uint16_t y) {+  return x < y ? y : x;+}++SCALAR_FUN_ATTR uint32_t umax32(uint32_t x, uint32_t y) {+  return x < y ? y : x;+}++SCALAR_FUN_ATTR uint64_t umax64(uint64_t x, uint64_t y) {+  return x < y ? y : x;+}++SCALAR_FUN_ATTR uint8_t shl8(uint8_t x, uint8_t y) {+  return (uint8_t)(x << y);+}++SCALAR_FUN_ATTR uint16_t shl16(uint16_t x, uint16_t y) {+  return (uint16_t)(x << y);+}++SCALAR_FUN_ATTR uint32_t shl32(uint32_t x, uint32_t y) {+  return x << y;+}++SCALAR_FUN_ATTR uint64_t shl64(uint64_t x, uint64_t y) {+  return x << y;+}++SCALAR_FUN_ATTR uint8_t lshr8(uint8_t x, uint8_t y) {+  return x >> y;+}++SCALAR_FUN_ATTR uint16_t lshr16(uint16_t x, uint16_t y) {+  return x >> y;+}++SCALAR_FUN_ATTR uint32_t lshr32(uint32_t x, uint32_t y) {+  return x >> y;+}++SCALAR_FUN_ATTR uint64_t lshr64(uint64_t x, uint64_t y) {+  return x >> y;+}++SCALAR_FUN_ATTR int8_t ashr8(int8_t x, int8_t y) {+  return x >> y;+}++SCALAR_FUN_ATTR int16_t ashr16(int16_t x, int16_t y) {+  return x >> y;+}++SCALAR_FUN_ATTR int32_t ashr32(int32_t x, int32_t y) {+  return x >> y;+}++SCALAR_FUN_ATTR int64_t ashr64(int64_t x, int64_t y) {+  return x >> y;+}++SCALAR_FUN_ATTR uint8_t and8(uint8_t x, uint8_t y) {+  return x & y;+}++SCALAR_FUN_ATTR uint16_t and16(uint16_t x, uint16_t y) {+  return x & y;+}++SCALAR_FUN_ATTR uint32_t and32(uint32_t x, uint32_t y) {+  return x & y;+}++SCALAR_FUN_ATTR uint64_t and64(uint64_t x, uint64_t y) {+  return x & y;+}++SCALAR_FUN_ATTR uint8_t or8(uint8_t x, uint8_t y) {+  return x | y;+}++SCALAR_FUN_ATTR uint16_t or16(uint16_t x, uint16_t y) {+  return x | y;+}++SCALAR_FUN_ATTR uint32_t or32(uint32_t x, uint32_t y) {+  return x | y;+}++SCALAR_FUN_ATTR uint64_t or64(uint64_t x, uint64_t y) {+  return x | y;+}++SCALAR_FUN_ATTR uint8_t xor8(uint8_t x, uint8_t y) {+  return x ^ y;+}++SCALAR_FUN_ATTR uint16_t xor16(uint16_t x, uint16_t y) {+  return x ^ y;+}++SCALAR_FUN_ATTR uint32_t xor32(uint32_t x, uint32_t y) {+  return x ^ y;+}++SCALAR_FUN_ATTR uint64_t xor64(uint64_t x, uint64_t y) {+  return x ^ y;+}++SCALAR_FUN_ATTR bool ult8(uint8_t x, uint8_t y) {+  return x < y;+}++SCALAR_FUN_ATTR bool ult16(uint16_t x, uint16_t y) {+  return x < y;+}++SCALAR_FUN_ATTR bool ult32(uint32_t x, uint32_t y) {+  return x < y;+}++SCALAR_FUN_ATTR bool ult64(uint64_t x, uint64_t y) {+  return x < y;+}++SCALAR_FUN_ATTR bool ule8(uint8_t x, uint8_t y) {+  return x <= y;+}++SCALAR_FUN_ATTR bool ule16(uint16_t x, uint16_t y) {+  return x <= y;+}++SCALAR_FUN_ATTR bool ule32(uint32_t x, uint32_t y) {+  return x <= y;+}++SCALAR_FUN_ATTR bool ule64(uint64_t x, uint64_t y) {+  return x <= y;+}++SCALAR_FUN_ATTR bool slt8(int8_t x, int8_t y) {+  return x < y;+}++SCALAR_FUN_ATTR bool slt16(int16_t x, int16_t y) {+  return x < y;+}++SCALAR_FUN_ATTR bool slt32(int32_t x, int32_t y) {+  return x < y;+}++SCALAR_FUN_ATTR bool slt64(int64_t x, int64_t y) {+  return x < y;+}++SCALAR_FUN_ATTR bool sle8(int8_t x, int8_t y) {+  return x <= y;+}++SCALAR_FUN_ATTR bool sle16(int16_t x, int16_t y) {+  return x <= y;+}++SCALAR_FUN_ATTR bool sle32(int32_t x, int32_t y) {+  return x <= y;+}++SCALAR_FUN_ATTR bool sle64(int64_t x, int64_t y) {+  return x <= y;+}++SCALAR_FUN_ATTR uint8_t pow8(uint8_t x, uint8_t y) {+  uint8_t res = 1, rem = y;++  while (rem != 0) {+    if (rem & 1)+      res *= x;+    rem >>= 1;+    x *= x;+  }+  return res;+}++SCALAR_FUN_ATTR uint16_t pow16(uint16_t x, uint16_t y) {+  uint16_t res = 1, rem = y;++  while (rem != 0) {+    if (rem & 1)+      res *= x;+    rem >>= 1;+    x *= x;+  }+  return res;+}++SCALAR_FUN_ATTR uint32_t pow32(uint32_t x, uint32_t y) {+  uint32_t res = 1, rem = y;++  while (rem != 0) {+    if (rem & 1)+      res *= x;+    rem >>= 1;+    x *= x;+  }+  return res;+}++SCALAR_FUN_ATTR uint64_t pow64(uint64_t x, uint64_t y) {+  uint64_t res = 1, rem = y;++  while (rem != 0) {+    if (rem & 1)+      res *= x;+    rem >>= 1;+    x *= x;+  }+  return res;+}++SCALAR_FUN_ATTR bool itob_i8_bool(int8_t x) {+  return x != 0;+}++SCALAR_FUN_ATTR bool itob_i16_bool(int16_t x) {+  return x != 0;+}++SCALAR_FUN_ATTR bool itob_i32_bool(int32_t x) {+  return x != 0;+}++SCALAR_FUN_ATTR bool itob_i64_bool(int64_t x) {+  return x != 0;+}++SCALAR_FUN_ATTR int8_t btoi_bool_i8(bool x) {+  return x;+}++SCALAR_FUN_ATTR int16_t btoi_bool_i16(bool x) {+  return x;+}++SCALAR_FUN_ATTR int32_t btoi_bool_i32(bool x) {+  return x;+}++SCALAR_FUN_ATTR int64_t btoi_bool_i64(bool x) {+  return x;+}++#define sext_i8_i8(x) ((int8_t) (int8_t) (x))+#define sext_i8_i16(x) ((int16_t) (int8_t) (x))+#define sext_i8_i32(x) ((int32_t) (int8_t) (x))+#define sext_i8_i64(x) ((int64_t) (int8_t) (x))+#define sext_i16_i8(x) ((int8_t) (int16_t) (x))+#define sext_i16_i16(x) ((int16_t) (int16_t) (x))+#define sext_i16_i32(x) ((int32_t) (int16_t) (x))+#define sext_i16_i64(x) ((int64_t) (int16_t) (x))+#define sext_i32_i8(x) ((int8_t) (int32_t) (x))+#define sext_i32_i16(x) ((int16_t) (int32_t) (x))+#define sext_i32_i32(x) ((int32_t) (int32_t) (x))+#define sext_i32_i64(x) ((int64_t) (int32_t) (x))+#define sext_i64_i8(x) ((int8_t) (int64_t) (x))+#define sext_i64_i16(x) ((int16_t) (int64_t) (x))+#define sext_i64_i32(x) ((int32_t) (int64_t) (x))+#define sext_i64_i64(x) ((int64_t) (int64_t) (x))+#define zext_i8_i8(x) ((int8_t) (uint8_t) (x))+#define zext_i8_i16(x) ((int16_t) (uint8_t) (x))+#define zext_i8_i32(x) ((int32_t) (uint8_t) (x))+#define zext_i8_i64(x) ((int64_t) (uint8_t) (x))+#define zext_i16_i8(x) ((int8_t) (uint16_t) (x))+#define zext_i16_i16(x) ((int16_t) (uint16_t) (x))+#define zext_i16_i32(x) ((int32_t) (uint16_t) (x))+#define zext_i16_i64(x) ((int64_t) (uint16_t) (x))+#define zext_i32_i8(x) ((int8_t) (uint32_t) (x))+#define zext_i32_i16(x) ((int16_t) (uint32_t) (x))+#define zext_i32_i32(x) ((int32_t) (uint32_t) (x))+#define zext_i32_i64(x) ((int64_t) (uint32_t) (x))+#define zext_i64_i8(x) ((int8_t) (uint64_t) (x))+#define zext_i64_i16(x) ((int16_t) (uint64_t) (x))+#define zext_i64_i32(x) ((int32_t) (uint64_t) (x))+#define zext_i64_i64(x) ((int64_t) (uint64_t) (x))++SCALAR_FUN_ATTR int8_t abs8(int8_t x) {+  return (int8_t)abs(x);+}++SCALAR_FUN_ATTR int16_t abs16(int16_t x) {+  return (int16_t)abs(x);+}++SCALAR_FUN_ATTR int32_t abs32(int32_t x) {+  return abs(x);+}++SCALAR_FUN_ATTR int64_t abs64(int64_t x) {+#if defined(__OPENCL_VERSION__) || defined(ISPC)+  return abs(x);+#else+  return llabs(x);+#endif+}++#if defined(__OPENCL_VERSION__)+SCALAR_FUN_ATTR int32_t futrts_popc8(int8_t x) {+  return popcount(x);+}++SCALAR_FUN_ATTR int32_t futrts_popc16(int16_t x) {+  return popcount(x);+}++SCALAR_FUN_ATTR int32_t futrts_popc32(int32_t x) {+  return popcount(x);+}++SCALAR_FUN_ATTR int32_t futrts_popc64(int64_t x) {+  return popcount(x);+}+#elif defined(__CUDA_ARCH__)++SCALAR_FUN_ATTR int32_t futrts_popc8(int8_t x) {+  return __popc(zext_i8_i32(x));+}++SCALAR_FUN_ATTR int32_t futrts_popc16(int16_t x) {+  return __popc(zext_i16_i32(x));+}++SCALAR_FUN_ATTR int32_t futrts_popc32(int32_t x) {+  return __popc(x);+}++SCALAR_FUN_ATTR int32_t futrts_popc64(int64_t x) {+  return __popcll(x);+}++#else // Not OpenCL or CUDA, but plain C.++SCALAR_FUN_ATTR int32_t futrts_popc8(uint8_t x) {+  int c = 0;+  for (; x; ++c) { x &= x - 1; }+  return c;+}++SCALAR_FUN_ATTR int32_t futrts_popc16(uint16_t x) {+  int c = 0;+  for (; x; ++c) { x &= x - 1; }+  return c;+}++SCALAR_FUN_ATTR int32_t futrts_popc32(uint32_t x) {+  int c = 0;+  for (; x; ++c) { x &= x - 1; }+  return c;+}++SCALAR_FUN_ATTR int32_t futrts_popc64(uint64_t x) {+  int c = 0;+  for (; x; ++c) { x &= x - 1; }+  return c;+}+#endif++#if defined(__OPENCL_VERSION__)+SCALAR_FUN_ATTR uint8_t  futrts_umul_hi8 ( uint8_t a,  uint8_t b) { return mul_hi(a, b); }+SCALAR_FUN_ATTR uint16_t futrts_umul_hi16(uint16_t a, uint16_t b) { return mul_hi(a, b); }+SCALAR_FUN_ATTR uint32_t futrts_umul_hi32(uint32_t a, uint32_t b) { return mul_hi(a, b); }+SCALAR_FUN_ATTR uint64_t futrts_umul_hi64(uint64_t a, uint64_t b) { return mul_hi(a, b); }+SCALAR_FUN_ATTR uint8_t  futrts_smul_hi8 ( int8_t a,  int8_t b) { return mul_hi(a, b); }+SCALAR_FUN_ATTR uint16_t futrts_smul_hi16(int16_t a, int16_t b) { return mul_hi(a, b); }+SCALAR_FUN_ATTR uint32_t futrts_smul_hi32(int32_t a, int32_t b) { return mul_hi(a, b); }+SCALAR_FUN_ATTR uint64_t futrts_smul_hi64(int64_t a, int64_t b) { return mul_hi(a, b); }+#elif defined(__CUDA_ARCH__)+SCALAR_FUN_ATTR  uint8_t futrts_umul_hi8(uint8_t a, uint8_t b) { return ((uint16_t)a) * ((uint16_t)b) >> 8; }+SCALAR_FUN_ATTR uint16_t futrts_umul_hi16(uint16_t a, uint16_t b) { return ((uint32_t)a) * ((uint32_t)b) >> 16; }+SCALAR_FUN_ATTR uint32_t futrts_umul_hi32(uint32_t a, uint32_t b) { return __umulhi(a, b); }+SCALAR_FUN_ATTR uint64_t futrts_umul_hi64(uint64_t a, uint64_t b) { return __umul64hi(a, b); }+SCALAR_FUN_ATTR  uint8_t futrts_smul_hi8 ( int8_t a, int8_t b) { return ((int16_t)a) * ((int16_t)b) >> 8; }+SCALAR_FUN_ATTR uint16_t futrts_smul_hi16(int16_t a, int16_t b) { return ((int32_t)a) * ((int32_t)b) >> 16; }+SCALAR_FUN_ATTR uint32_t futrts_smul_hi32(int32_t a, int32_t b) { return __mulhi(a, b); }+SCALAR_FUN_ATTR uint64_t futrts_smul_hi64(int64_t a, int64_t b) { return __mul64hi(a, b); }+#elif ISPC+SCALAR_FUN_ATTR uint8_t futrts_umul_hi8(uint8_t a, uint8_t b) { return ((uint16_t)a) * ((uint16_t)b) >> 8; }+SCALAR_FUN_ATTR uint16_t futrts_umul_hi16(uint16_t a, uint16_t b) { return ((uint32_t)a) * ((uint32_t)b) >> 16; }+SCALAR_FUN_ATTR uint32_t futrts_umul_hi32(uint32_t a, uint32_t b) { return ((uint64_t)a) * ((uint64_t)b) >> 32; }+SCALAR_FUN_ATTR uint64_t futrts_umul_hi64(uint64_t a, uint64_t b) {+  uint64_t ah = a >> 32;+  uint64_t al = a & 0xffffffff;+  uint64_t bh = b >> 32;+  uint64_t bl = b & 0xffffffff;++  uint64_t p1 = al * bl;+  uint64_t p2 = al * bh;+  uint64_t p3 = ah * bl;+  uint64_t p4 = ah * bh;++  uint64_t p1h = p1 >> 32;+  uint64_t p2h = p2 >> 32;+  uint64_t p3h = p3 >> 32;+  uint64_t p2l = p2 & 0xffffffff;+  uint64_t p3l = p3 & 0xffffffff;++  uint64_t l = p1h + p2l + p3l;+  uint64_t m = (p2 >> 32) + (p3 >> 32);+  uint64_t h = (l >> 32) + m + p4;++  return h;+}+SCALAR_FUN_ATTR  int8_t futrts_smul_hi8 ( int8_t a,  int8_t b) { return ((uint16_t)a) * ((uint16_t)b) >> 8; }+SCALAR_FUN_ATTR int16_t futrts_smul_hi16(int16_t a, int16_t b) { return ((uint32_t)a) * ((uint32_t)b) >> 16; }+SCALAR_FUN_ATTR int32_t futrts_smul_hi32(int32_t a, int32_t b) { return ((uint64_t)a) * ((uint64_t)b) >> 32; }+SCALAR_FUN_ATTR int64_t futrts_smul_hi64(int64_t a, int64_t b) {+  uint64_t ah = a >> 32;+  uint64_t al = a & 0xffffffff;+  uint64_t bh = b >> 32;+  uint64_t bl = b & 0xffffffff;++  uint64_t p1 =  al * bl;+  int64_t  p2 = al * bh;+  int64_t  p3 = ah * bl;+  uint64_t p4 =  ah * bh;++  uint64_t p1h = p1 >> 32;+  uint64_t p2h = p2 >> 32;+  uint64_t p3h = p3 >> 32;+  uint64_t p2l = p2 & 0xffffffff;+  uint64_t p3l = p3 & 0xffffffff;++  uint64_t l = p1h + p2l + p3l;+  uint64_t m = (p2 >> 32) + (p3 >> 32);+  uint64_t h = (l >> 32) + m + p4;++  return h;+}++#else // Not OpenCL, ISPC, or CUDA, but plain C.+SCALAR_FUN_ATTR uint8_t futrts_umul_hi8(uint8_t a, uint8_t b) { return ((uint16_t)a) * ((uint16_t)b) >> 8; }+SCALAR_FUN_ATTR uint16_t futrts_umul_hi16(uint16_t a, uint16_t b) { return ((uint32_t)a) * ((uint32_t)b) >> 16; }+SCALAR_FUN_ATTR uint32_t futrts_umul_hi32(uint32_t a, uint32_t b) { return ((uint64_t)a) * ((uint64_t)b) >> 32; }+SCALAR_FUN_ATTR uint64_t futrts_umul_hi64(uint64_t a, uint64_t b) { return ((__uint128_t)a) * ((__uint128_t)b) >> 64; }+SCALAR_FUN_ATTR int8_t futrts_smul_hi8(int8_t a, int8_t b) { return ((int16_t)a) * ((int16_t)b) >> 8; }+SCALAR_FUN_ATTR int16_t futrts_smul_hi16(int16_t a, int16_t b) { return ((int32_t)a) * ((int32_t)b) >> 16; }+SCALAR_FUN_ATTR int32_t futrts_smul_hi32(int32_t a, int32_t b) { return ((int64_t)a) * ((int64_t)b) >> 32; }+SCALAR_FUN_ATTR int64_t futrts_smul_hi64(int64_t a, int64_t b) { return ((__int128_t)a) * ((__int128_t)b) >> 64; }+#endif++#if defined(__OPENCL_VERSION__)+SCALAR_FUN_ATTR  uint8_t futrts_umad_hi8 ( uint8_t a,  uint8_t b,  uint8_t c) { return mad_hi(a, b, c); }+SCALAR_FUN_ATTR uint16_t futrts_umad_hi16(uint16_t a, uint16_t b, uint16_t c) { return mad_hi(a, b, c); }+SCALAR_FUN_ATTR uint32_t futrts_umad_hi32(uint32_t a, uint32_t b, uint32_t c) { return mad_hi(a, b, c); }+SCALAR_FUN_ATTR uint64_t futrts_umad_hi64(uint64_t a, uint64_t b, uint64_t c) { return mad_hi(a, b, c); }+SCALAR_FUN_ATTR  uint8_t futrts_smad_hi8( int8_t a,  int8_t b,   int8_t c) { return mad_hi(a, b, c); }+SCALAR_FUN_ATTR uint16_t futrts_smad_hi16(int16_t a, int16_t b, int16_t c) { return mad_hi(a, b, c); }+SCALAR_FUN_ATTR uint32_t futrts_smad_hi32(int32_t a, int32_t b, int32_t c) { return mad_hi(a, b, c); }+SCALAR_FUN_ATTR uint64_t futrts_smad_hi64(int64_t a, int64_t b, int64_t c) { return mad_hi(a, b, c); }+#else // Not OpenCL++SCALAR_FUN_ATTR  uint8_t futrts_umad_hi8( uint8_t a,  uint8_t b,  uint8_t c) { return futrts_umul_hi8(a, b) + c; }+SCALAR_FUN_ATTR uint16_t futrts_umad_hi16(uint16_t a, uint16_t b, uint16_t c) { return futrts_umul_hi16(a, b) + c; }+SCALAR_FUN_ATTR uint32_t futrts_umad_hi32(uint32_t a, uint32_t b, uint32_t c) { return futrts_umul_hi32(a, b) + c; }+SCALAR_FUN_ATTR uint64_t futrts_umad_hi64(uint64_t a, uint64_t b, uint64_t c) { return futrts_umul_hi64(a, b) + c; }+SCALAR_FUN_ATTR  uint8_t futrts_smad_hi8 ( int8_t a,  int8_t b,  int8_t c) { return futrts_smul_hi8(a, b) + c; }+SCALAR_FUN_ATTR uint16_t futrts_smad_hi16(int16_t a, int16_t b, int16_t c) { return futrts_smul_hi16(a, b) + c; }+SCALAR_FUN_ATTR uint32_t futrts_smad_hi32(int32_t a, int32_t b, int32_t c) { return futrts_smul_hi32(a, b) + c; }+SCALAR_FUN_ATTR uint64_t futrts_smad_hi64(int64_t a, int64_t b, int64_t c) { return futrts_smul_hi64(a, b) + c; }+#endif++#if defined(__OPENCL_VERSION__)+SCALAR_FUN_ATTR int32_t futrts_clzz8(int8_t x) {+  return clz(x);+}++SCALAR_FUN_ATTR int32_t futrts_clzz16(int16_t x) {+  return clz(x);+}++SCALAR_FUN_ATTR int32_t futrts_clzz32(int32_t x) {+  return clz(x);+}++SCALAR_FUN_ATTR int32_t futrts_clzz64(int64_t x) {+  return clz(x);+}++#elif defined(__CUDA_ARCH__)++SCALAR_FUN_ATTR int32_t futrts_clzz8(int8_t x) {+  return __clz(zext_i8_i32(x)) - 24;+}++SCALAR_FUN_ATTR int32_t futrts_clzz16(int16_t x) {+  return __clz(zext_i16_i32(x)) - 16;+}++SCALAR_FUN_ATTR int32_t futrts_clzz32(int32_t x) {+  return __clz(x);+}++SCALAR_FUN_ATTR int32_t futrts_clzz64(int64_t x) {+  return __clzll(x);+}++#elif ISPC++SCALAR_FUN_ATTR int32_t futrts_clzz8(int8_t x) {+  return count_leading_zeros((int32_t)(uint8_t)x)-24;+}++SCALAR_FUN_ATTR int32_t futrts_clzz16(int16_t x) {+  return count_leading_zeros((int32_t)(uint16_t)x)-16;+}++SCALAR_FUN_ATTR int32_t futrts_clzz32(int32_t x) {+  return count_leading_zeros(x);+}++SCALAR_FUN_ATTR int32_t futrts_clzz64(int64_t x) {+  return count_leading_zeros(x);+}++#else // Not OpenCL, ISPC or CUDA, but plain C.++SCALAR_FUN_ATTR int32_t futrts_clzz8(int8_t x) {+  return x == 0 ? 8 : __builtin_clz((uint32_t)zext_i8_i32(x)) - 24;+}++SCALAR_FUN_ATTR int32_t futrts_clzz16(int16_t x) {+  return x == 0 ? 16 : __builtin_clz((uint32_t)zext_i16_i32(x)) - 16;+}++SCALAR_FUN_ATTR int32_t futrts_clzz32(int32_t x) {+  return x == 0 ? 32 : __builtin_clz((uint32_t)x);+}++SCALAR_FUN_ATTR int32_t futrts_clzz64(int64_t x) {+  return x == 0 ? 64 : __builtin_clzll((uint64_t)x);+}+#endif++#if defined(__OPENCL_VERSION__)+SCALAR_FUN_ATTR int32_t futrts_ctzz8(int8_t x) {+  int i = 0;+  for (; i < 8 && (x & 1) == 0; i++, x >>= 1)+    ;+  return i;+}++SCALAR_FUN_ATTR int32_t futrts_ctzz16(int16_t x) {+  int i = 0;+  for (; i < 16 && (x & 1) == 0; i++, x >>= 1)+    ;+  return i;+}++SCALAR_FUN_ATTR int32_t futrts_ctzz32(int32_t x) {+  int i = 0;+  for (; i < 32 && (x & 1) == 0; i++, x >>= 1)+    ;+  return i;+}++SCALAR_FUN_ATTR int32_t futrts_ctzz64(int64_t x) {+  int i = 0;+  for (; i < 64 && (x & 1) == 0; i++, x >>= 1)+    ;+  return i;+}++#elif defined(__CUDA_ARCH__)++SCALAR_FUN_ATTR int32_t futrts_ctzz8(int8_t x) {+  int y = __ffs(x);+  return y == 0 ? 8 : y - 1;+}++SCALAR_FUN_ATTR int32_t futrts_ctzz16(int16_t x) {+  int y = __ffs(x);+  return y == 0 ? 16 : y - 1;+}++SCALAR_FUN_ATTR int32_t futrts_ctzz32(int32_t x) {+  int y = __ffs(x);+  return y == 0 ? 32 : y - 1;+}++SCALAR_FUN_ATTR int32_t futrts_ctzz64(int64_t x) {+  int y = __ffsll(x);+  return y == 0 ? 64 : y - 1;+}++#elif ISPC++SCALAR_FUN_ATTR int32_t futrts_ctzz8(int8_t x) {+  return x == 0 ? 8 : count_trailing_zeros((int32_t)x);+}++SCALAR_FUN_ATTR int32_t futrts_ctzz16(int16_t x) {+  return x == 0 ? 16 : count_trailing_zeros((int32_t)x);+}++SCALAR_FUN_ATTR int32_t futrts_ctzz32(int32_t x) {+  return count_trailing_zeros(x);+}++SCALAR_FUN_ATTR int32_t futrts_ctzz64(int64_t x) {+  return count_trailing_zeros(x);+}++#else // Not OpenCL or CUDA, but plain C.++SCALAR_FUN_ATTR int32_t futrts_ctzz8(int8_t x) {+  return x == 0 ? 8 : __builtin_ctz((uint32_t)x);+}++SCALAR_FUN_ATTR int32_t futrts_ctzz16(int16_t x) {+  return x == 0 ? 16 : __builtin_ctz((uint32_t)x);+}++SCALAR_FUN_ATTR int32_t futrts_ctzz32(int32_t x) {+  return x == 0 ? 32 : __builtin_ctz((uint32_t)x);+}++SCALAR_FUN_ATTR int32_t futrts_ctzz64(int64_t x) {+  return x == 0 ? 64 : __builtin_ctzll((uint64_t)x);+}+#endif++SCALAR_FUN_ATTR float fdiv32(float x, float y) {+  return x / y;+}++SCALAR_FUN_ATTR float fadd32(float x, float y) {+  return x + y;+}++SCALAR_FUN_ATTR float fsub32(float x, float y) {+  return x - y;+}++SCALAR_FUN_ATTR float fmul32(float x, float y) {+  return x * y;+}++SCALAR_FUN_ATTR bool cmplt32(float x, float y) {+  return x < y;+}++SCALAR_FUN_ATTR bool cmple32(float x, float y) {+  return x <= y;+}++SCALAR_FUN_ATTR float sitofp_i8_f32(int8_t x) {+  return (float) x;+}++SCALAR_FUN_ATTR float sitofp_i16_f32(int16_t x) {+  return (float) x;+}++SCALAR_FUN_ATTR float sitofp_i32_f32(int32_t x) {+  return (float) x;+}++SCALAR_FUN_ATTR float sitofp_i64_f32(int64_t x) {+  return (float) x;+}++SCALAR_FUN_ATTR float uitofp_i8_f32(uint8_t x) {+  return (float) x;+}++SCALAR_FUN_ATTR float uitofp_i16_f32(uint16_t x) {+  return (float) x;+}++SCALAR_FUN_ATTR float uitofp_i32_f32(uint32_t x) {+  return (float) x;+}++SCALAR_FUN_ATTR float uitofp_i64_f32(uint64_t x) {+  return (float) x;+}++#ifdef __OPENCL_VERSION__+SCALAR_FUN_ATTR float fabs32(float x) {+  return fabs(x);+}++SCALAR_FUN_ATTR float fmax32(float x, float y) {+  return fmax(x, y);+}++SCALAR_FUN_ATTR float fmin32(float x, float y) {+  return fmin(x, y);+}++SCALAR_FUN_ATTR float fpow32(float x, float y) {+  return pow(x, y);+}++#elif ISPC++SCALAR_FUN_ATTR float fabs32(float x) {+  return abs(x);+}++SCALAR_FUN_ATTR float fmax32(float x, float y) {+  return isnan(x) ? y : isnan(y) ? x : max(x, y);+}++SCALAR_FUN_ATTR float fmin32(float x, float y) {+  return isnan(x) ? y : isnan(y) ? x : min(x, y);+}++SCALAR_FUN_ATTR float fpow32(float a, float b) {+  float ret;+  foreach_active (i) {+      uniform float r = __stdlib_powf(extract(a, i), extract(b, i));+      ret = insert(ret, i, r);+  }+  return ret;+}++#else // Not OpenCL, but CUDA or plain C.++SCALAR_FUN_ATTR float fabs32(float x) {+  return fabsf(x);+}++SCALAR_FUN_ATTR float fmax32(float x, float y) {+  return fmaxf(x, y);+}++SCALAR_FUN_ATTR float fmin32(float x, float y) {+  return fminf(x, y);+}++SCALAR_FUN_ATTR float fpow32(float x, float y) {+  return powf(x, y);+}+#endif++SCALAR_FUN_ATTR bool futrts_isnan32(float x) {+  return isnan(x);+}++#if ISPC++SCALAR_FUN_ATTR bool futrts_isinf32(float x) {+  return !isnan(x) && isnan(x - x);+}++SCALAR_FUN_ATTR bool futrts_isfinite32(float x) {+  return !isnan(x) && !futrts_isinf32(x);+}++#else++SCALAR_FUN_ATTR bool futrts_isinf32(float x) {+  return isinf(x);+}++#endif++SCALAR_FUN_ATTR int8_t fptosi_f32_i8(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (int8_t) x;+  }+}++SCALAR_FUN_ATTR int16_t fptosi_f32_i16(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (int16_t) x;+  }+}++SCALAR_FUN_ATTR int32_t fptosi_f32_i32(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (int32_t) x;+  }+}++SCALAR_FUN_ATTR int64_t fptosi_f32_i64(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (int64_t) x;+  };+}++SCALAR_FUN_ATTR uint8_t fptoui_f32_i8(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (uint8_t) (int8_t) x;+  }+}++SCALAR_FUN_ATTR uint16_t fptoui_f32_i16(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (uint16_t) (int16_t) x;+  }+}++SCALAR_FUN_ATTR uint32_t fptoui_f32_i32(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (uint32_t) (int32_t) x;+  }+}++SCALAR_FUN_ATTR uint64_t fptoui_f32_i64(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (uint64_t) (int64_t) x;+  }+}++SCALAR_FUN_ATTR bool ftob_f32_bool(float x) {+  return x != 0;+}++SCALAR_FUN_ATTR float btof_bool_f32(bool x) {+  return x ? 1 : 0;+}++#ifdef __OPENCL_VERSION__+SCALAR_FUN_ATTR float futrts_log32(float x) {+  return log(x);+}++SCALAR_FUN_ATTR float futrts_log2_32(float x) {+  return log2(x);+}++SCALAR_FUN_ATTR float futrts_log10_32(float x) {+  return log10(x);+}++SCALAR_FUN_ATTR float futrts_log1p_32(float x) {+  return log1p(x);+}++SCALAR_FUN_ATTR float futrts_sqrt32(float x) {+  return sqrt(x);+}++SCALAR_FUN_ATTR float futrts_cbrt32(float x) {+  return cbrt(x);+}++SCALAR_FUN_ATTR float futrts_exp32(float x) {+  return exp(x);+}++SCALAR_FUN_ATTR float futrts_cos32(float x) {+  return cos(x);+}++SCALAR_FUN_ATTR float futrts_sin32(float x) {+  return sin(x);+}++SCALAR_FUN_ATTR float futrts_tan32(float x) {+  return tan(x);+}++SCALAR_FUN_ATTR float futrts_acos32(float x) {+  return acos(x);+}++SCALAR_FUN_ATTR float futrts_asin32(float x) {+  return asin(x);+}++SCALAR_FUN_ATTR float futrts_atan32(float x) {+  return atan(x);+}++SCALAR_FUN_ATTR float futrts_cosh32(float x) {+  return cosh(x);+}++SCALAR_FUN_ATTR float futrts_sinh32(float x) {+  return sinh(x);+}++SCALAR_FUN_ATTR float futrts_tanh32(float x) {+  return tanh(x);+}++SCALAR_FUN_ATTR float futrts_acosh32(float x) {+  return acosh(x);+}++SCALAR_FUN_ATTR float futrts_asinh32(float x) {+  return asinh(x);+}++SCALAR_FUN_ATTR float futrts_atanh32(float x) {+  return atanh(x);+}++SCALAR_FUN_ATTR float futrts_atan2_32(float x, float y) {+  return atan2(x, y);+}++SCALAR_FUN_ATTR float futrts_hypot32(float x, float y) {+  return hypot(x, y);+}++SCALAR_FUN_ATTR float futrts_gamma32(float x) {+  return tgamma(x);+}++SCALAR_FUN_ATTR float futrts_lgamma32(float x) {+  return lgamma(x);+}++SCALAR_FUN_ATTR float futrts_erf32(float x) {+  return erf(x);+}++SCALAR_FUN_ATTR float futrts_erfc32(float x) {+  return erfc(x);+}++SCALAR_FUN_ATTR float fmod32(float x, float y) {+  return fmod(x, y);+}++SCALAR_FUN_ATTR float futrts_round32(float x) {+  return rint(x);+}++SCALAR_FUN_ATTR float futrts_floor32(float x) {+  return floor(x);+}++SCALAR_FUN_ATTR float futrts_ceil32(float x) {+  return ceil(x);+}++SCALAR_FUN_ATTR float futrts_nextafter32(float x, float y) {+  return nextafter(x, y);+}++SCALAR_FUN_ATTR float futrts_lerp32(float v0, float v1, float t) {+  return mix(v0, v1, t);+}++SCALAR_FUN_ATTR float futrts_mad32(float a, float b, float c) {+  return mad(a, b, c);+}++SCALAR_FUN_ATTR float futrts_fma32(float a, float b, float c) {+  return fma(a, b, c);+}++#elif ISPC++SCALAR_FUN_ATTR float futrts_log32(float x) {+  return futrts_isfinite32(x) || (futrts_isinf32(x) && x < 0)? log(x) : x;+}++SCALAR_FUN_ATTR float futrts_log2_32(float x) {+  return futrts_log32(x) / log(2.0f);+}++SCALAR_FUN_ATTR float futrts_log10_32(float x) {+  return futrts_log32(x) / log(10.0f);+}++SCALAR_FUN_ATTR float futrts_log1p_32(float x) {+  if(x == -1.0f || (futrts_isinf32(x) && x > 0.0f)) return x / 0.0f;+  float y = 1.0f + x;+  float z = y - 1.0f;+  return log(y) - (z-x)/y;+}++SCALAR_FUN_ATTR float futrts_sqrt32(float x) {+  return sqrt(x);+}++extern "C" unmasked uniform float cbrtf(uniform float);+SCALAR_FUN_ATTR float futrts_cbrt32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = cbrtf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++SCALAR_FUN_ATTR float futrts_exp32(float x) {+  return exp(x);+}++SCALAR_FUN_ATTR float futrts_cos32(float x) {+  return cos(x);+}++SCALAR_FUN_ATTR float futrts_sin32(float x) {+  return sin(x);+}++SCALAR_FUN_ATTR float futrts_tan32(float x) {+  return tan(x);+}++SCALAR_FUN_ATTR float futrts_acos32(float x) {+  return acos(x);+}++SCALAR_FUN_ATTR float futrts_asin32(float x) {+  return asin(x);+}++SCALAR_FUN_ATTR float futrts_atan32(float x) {+  return atan(x);+}++SCALAR_FUN_ATTR float futrts_cosh32(float x) {+  return (exp(x)+exp(-x)) / 2.0f;+}++SCALAR_FUN_ATTR float futrts_sinh32(float x) {+  return (exp(x)-exp(-x)) / 2.0f;+}++SCALAR_FUN_ATTR float futrts_tanh32(float x) {+  return futrts_sinh32(x)/futrts_cosh32(x);+}++SCALAR_FUN_ATTR float futrts_acosh32(float x) {+  float f = x+sqrt(x*x-1);+  if(futrts_isfinite32(f)) return log(f);+  return f;+}++SCALAR_FUN_ATTR float futrts_asinh32(float x) {+  float f = x+sqrt(x*x+1);+  if(futrts_isfinite32(f)) return log(f);+  return f;++}++SCALAR_FUN_ATTR float futrts_atanh32(float x) {+  float f = (1+x)/(1-x);+  if(futrts_isfinite32(f)) return log(f)/2.0f;+  return f;++}++SCALAR_FUN_ATTR float futrts_atan2_32(float x, float y) {+  return (x == 0.0f && y == 0.0f) ? 0.0f : atan2(x, y);+}++SCALAR_FUN_ATTR float futrts_hypot32(float x, float y) {+  if (futrts_isfinite32(x) && futrts_isfinite32(y)) {+    x = abs(x);+    y = abs(y);+    float a;+    float b;+    if (x >= y){+        a = x;+        b = y;+    } else {+        a = y;+        b = x;+    }+    if(b == 0){+      return a;+    }++    int e;+    float an;+    float bn;+    an = frexp (a, &e);+    bn = ldexp (b, - e);+    float cn;+    cn = sqrt (an * an + bn * bn);+    return ldexp (cn, e);+  } else {+    if (futrts_isinf32(x) || futrts_isinf32(y)) return INFINITY;+    else return x + y;+  }++}++extern "C" unmasked uniform float tgammaf(uniform float x);+SCALAR_FUN_ATTR float futrts_gamma32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = tgammaf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform float lgammaf(uniform float x);+SCALAR_FUN_ATTR float futrts_lgamma32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = lgammaf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform float erff(uniform float x);+SCALAR_FUN_ATTR float futrts_erf32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = erff(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform float erfcf(uniform float x);+SCALAR_FUN_ATTR float futrts_erfc32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = erfcf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++SCALAR_FUN_ATTR float fmod32(float x, float y) {+  return x - y * trunc(x/y);+}++SCALAR_FUN_ATTR float futrts_round32(float x) {+  return round(x);+}++SCALAR_FUN_ATTR float futrts_floor32(float x) {+  return floor(x);+}++SCALAR_FUN_ATTR float futrts_ceil32(float x) {+  return ceil(x);+}++extern "C" unmasked uniform float nextafterf(uniform float x, uniform float y);+SCALAR_FUN_ATTR float futrts_nextafter32(float x, float y) {+  float res;+  foreach_active (i) {+    uniform float r = nextafterf(extract(x, i), extract(y, i));+    res = insert(res, i, r);+  }+  return res;+}++SCALAR_FUN_ATTR float futrts_lerp32(float v0, float v1, float t) {+  return v0 + (v1 - v0) * t;+}++SCALAR_FUN_ATTR float futrts_mad32(float a, float b, float c) {+  return a * b + c;+}++SCALAR_FUN_ATTR float futrts_fma32(float a, float b, float c) {+  return a * b + c;+}++#else // Not OpenCL or ISPC, but CUDA or plain C.++SCALAR_FUN_ATTR float futrts_log32(float x) {+  return logf(x);+}++SCALAR_FUN_ATTR float futrts_log2_32(float x) {+  return log2f(x);+}++SCALAR_FUN_ATTR float futrts_log10_32(float x) {+  return log10f(x);+}++SCALAR_FUN_ATTR float futrts_log1p_32(float x) {+  return log1pf(x);+}++SCALAR_FUN_ATTR float futrts_sqrt32(float x) {+  return sqrtf(x);+}++SCALAR_FUN_ATTR float futrts_cbrt32(float x) {+  return cbrtf(x);+}++SCALAR_FUN_ATTR float futrts_exp32(float x) {+  return expf(x);+}++SCALAR_FUN_ATTR float futrts_cos32(float x) {+  return cosf(x);+}++SCALAR_FUN_ATTR float futrts_sin32(float x) {+  return sinf(x);+}++SCALAR_FUN_ATTR float futrts_tan32(float x) {+  return tanf(x);+}++SCALAR_FUN_ATTR float futrts_acos32(float x) {+  return acosf(x);+}++SCALAR_FUN_ATTR float futrts_asin32(float x) {+  return asinf(x);+}++SCALAR_FUN_ATTR float futrts_atan32(float x) {+  return atanf(x);+}++SCALAR_FUN_ATTR float futrts_cosh32(float x) {+  return coshf(x);+}++SCALAR_FUN_ATTR float futrts_sinh32(float x) {+  return sinhf(x);+}++SCALAR_FUN_ATTR float futrts_tanh32(float x) {+  return tanhf(x);+}++SCALAR_FUN_ATTR float futrts_acosh32(float x) {+  return acoshf(x);+}++SCALAR_FUN_ATTR float futrts_asinh32(float x) {+  return asinhf(x);+}++SCALAR_FUN_ATTR float futrts_atanh32(float x) {+  return atanhf(x);+}++SCALAR_FUN_ATTR float futrts_atan2_32(float x, float y) {+  return atan2f(x, y);+}++SCALAR_FUN_ATTR float futrts_hypot32(float x, float y) {+  return hypotf(x, y);+}++SCALAR_FUN_ATTR float futrts_gamma32(float x) {+  return tgammaf(x);+}++SCALAR_FUN_ATTR float futrts_lgamma32(float x) {+  return lgammaf(x);+}++SCALAR_FUN_ATTR float futrts_erf32(float x) {+  return erff(x);+}++SCALAR_FUN_ATTR float futrts_erfc32(float x) {+  return erfcf(x);+}++SCALAR_FUN_ATTR float fmod32(float x, float y) {+  return fmodf(x, y);+}++SCALAR_FUN_ATTR float futrts_round32(float x) {+  return rintf(x);+}++SCALAR_FUN_ATTR float futrts_floor32(float x) {+  return floorf(x);+}++SCALAR_FUN_ATTR float futrts_ceil32(float x) {+  return ceilf(x);+}++SCALAR_FUN_ATTR float futrts_nextafter32(float x, float y) {+  return nextafterf(x, y);+}++SCALAR_FUN_ATTR float futrts_lerp32(float v0, float v1, float t) {+  return v0 + (v1 - v0) * t;+}++SCALAR_FUN_ATTR float futrts_mad32(float a, float b, float c) {+  return a * b + c;+}++SCALAR_FUN_ATTR float futrts_fma32(float a, float b, float c) {+  return fmaf(a, b, c);+}+#endif++#if ISPC+SCALAR_FUN_ATTR int32_t futrts_to_bits32(float x) {+  return intbits(x);+}++SCALAR_FUN_ATTR float futrts_from_bits32(int32_t x) {+  return floatbits(x);+}+#else+SCALAR_FUN_ATTR int32_t futrts_to_bits32(float x) {+  union {+    float f;+    int32_t t;+  } p;++  p.f = x;+  return p.t;+}++SCALAR_FUN_ATTR float futrts_from_bits32(int32_t x) {+  union {+    int32_t f;+    float t;+  } p;++  p.f = x;+  return p.t;+}+#endif++SCALAR_FUN_ATTR float fsignum32(float x) {+  return futrts_isnan32(x) ? x : (x > 0 ? 1 : 0) - (x < 0 ? 1 : 0);+}++#ifdef FUTHARK_F64_ENABLED++#if ISPC+SCALAR_FUN_ATTR bool futrts_isinf64(float x) {+  return !isnan(x) && isnan(x - x);+}++SCALAR_FUN_ATTR bool futrts_isfinite64(float x) {+  return !isnan(x) && !futrts_isinf64(x);+}++SCALAR_FUN_ATTR double fdiv64(double x, double y) {+  return x / y;+}++SCALAR_FUN_ATTR double fadd64(double x, double y) {+  return x + y;+}++SCALAR_FUN_ATTR double fsub64(double x, double y) {+  return x - y;+}++SCALAR_FUN_ATTR double fmul64(double x, double y) {+  return x * y;+}++SCALAR_FUN_ATTR bool cmplt64(double x, double y) {+  return x < y;+}++SCALAR_FUN_ATTR bool cmple64(double x, double y) {+  return x <= y;+}++SCALAR_FUN_ATTR double sitofp_i8_f64(int8_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double sitofp_i16_f64(int16_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double sitofp_i32_f64(int32_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double sitofp_i64_f64(int64_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double uitofp_i8_f64(uint8_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double uitofp_i16_f64(uint16_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double uitofp_i32_f64(uint32_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double uitofp_i64_f64(uint64_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double fabs64(double x) {+  return abs(x);+}++SCALAR_FUN_ATTR double fmax64(double x, double y) {+  return isnan(x) ? y : isnan(y) ? x : max(x, y);+}++SCALAR_FUN_ATTR double fmin64(double x, double y) {+  return isnan(x) ? y : isnan(y) ? x : min(x, y);+}++SCALAR_FUN_ATTR double fpow64(double a, double b) {+  float ret;+  foreach_active (i) {+      uniform float r = __stdlib_powf(extract(a, i), extract(b, i));+      ret = insert(ret, i, r);+  }+  return ret;+}++SCALAR_FUN_ATTR double futrts_log64(double x) {+  return futrts_isfinite64(x) || (futrts_isinf64(x) && x < 0)? log(x) : x;+}++SCALAR_FUN_ATTR double futrts_log2_64(double x) {+  return futrts_log64(x)/log(2.0d);+}++SCALAR_FUN_ATTR double futrts_log10_64(double x) {+  return futrts_log64(x)/log(10.0d);+}++SCALAR_FUN_ATTR double futrts_log1p_64(double x) {+  if(x == -1.0d || (futrts_isinf64(x) && x > 0.0d)) return x / 0.0d;+  double y = 1.0d + x;+  double z = y - 1.0d;+  return log(y) - (z-x)/y;+}++SCALAR_FUN_ATTR double futrts_sqrt64(double x) {+  return sqrt(x);+}++extern "C" unmasked uniform double cbrt(uniform double);+SCALAR_FUN_ATTR double futrts_cbrt64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = cbrtf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++SCALAR_FUN_ATTR double futrts_exp64(double x) {+  return exp(x);+}++SCALAR_FUN_ATTR double futrts_cos64(double x) {+  return cos(x);+}++SCALAR_FUN_ATTR double futrts_sin64(double x) {+  return sin(x);+}++SCALAR_FUN_ATTR double futrts_tan64(double x) {+  return tan(x);+}++SCALAR_FUN_ATTR double futrts_acos64(double x) {+  return acos(x);+}++SCALAR_FUN_ATTR double futrts_asin64(double x) {+  return asin(x);+}++SCALAR_FUN_ATTR double futrts_atan64(double x) {+  return atan(x);+}++SCALAR_FUN_ATTR double futrts_cosh64(double x) {+  return (exp(x)+exp(-x)) / 2.0d;+}++SCALAR_FUN_ATTR double futrts_sinh64(double x) {+  return (exp(x)-exp(-x)) / 2.0d;+}++SCALAR_FUN_ATTR double futrts_tanh64(double x) {+  return futrts_sinh64(x)/futrts_cosh64(x);+}++SCALAR_FUN_ATTR double futrts_acosh64(double x) {+  double f = x+sqrt(x*x-1.0d);+  if(futrts_isfinite64(f)) return log(f);+  return f;+}++SCALAR_FUN_ATTR double futrts_asinh64(double x) {+  double f = x+sqrt(x*x+1.0d);+  if(futrts_isfinite64(f)) return log(f);+  return f;+}++SCALAR_FUN_ATTR double futrts_atanh64(double x) {+  double f = (1.0d+x)/(1.0d-x);+  if(futrts_isfinite64(f)) return log(f)/2.0d;+  return f;++}++SCALAR_FUN_ATTR double futrts_atan2_64(double x, double y) {+  return atan2(x, y);+}++extern "C" unmasked uniform double hypot(uniform double x, uniform double y);+SCALAR_FUN_ATTR double futrts_hypot64(double x, double y) {+  double res;+  foreach_active (i) {+    uniform double r = hypot(extract(x, i), extract(y, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform double tgamma(uniform double x);+SCALAR_FUN_ATTR double futrts_gamma64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = tgamma(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform double lgamma(uniform double x);+SCALAR_FUN_ATTR double futrts_lgamma64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = lgamma(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform double erf(uniform double x);+SCALAR_FUN_ATTR double futrts_erf64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = erf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform double erfc(uniform double x);+SCALAR_FUN_ATTR double futrts_erfc64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = erfc(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++SCALAR_FUN_ATTR double futrts_fma64(double a, double b, double c) {+  return a * b + c;+}++SCALAR_FUN_ATTR double futrts_round64(double x) {+  return round(x);+}++SCALAR_FUN_ATTR double futrts_ceil64(double x) {+  return ceil(x);+}++extern "C" unmasked uniform double nextafter(uniform float x, uniform double y);+SCALAR_FUN_ATTR float futrts_nextafter64(double x, double y) {+  double res;+  foreach_active (i) {+    uniform double r = nextafter(extract(x, i), extract(y, i));+    res = insert(res, i, r);+  }+  return res;+}++SCALAR_FUN_ATTR double futrts_floor64(double x) {+  return floor(x);+}++SCALAR_FUN_ATTR bool futrts_isnan64(double x) {+  return isnan(x);+}++SCALAR_FUN_ATTR int8_t fptosi_f64_i8(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int8_t) x;+  }+}++SCALAR_FUN_ATTR int16_t fptosi_f64_i16(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int16_t) x;+  }+}++SCALAR_FUN_ATTR int32_t fptosi_f64_i32(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int32_t) x;+  }+}++SCALAR_FUN_ATTR int64_t fptosi_f64_i64(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int64_t) x;+  }+}++SCALAR_FUN_ATTR uint8_t fptoui_f64_i8(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint8_t) (int8_t) x;+  }+}++SCALAR_FUN_ATTR uint16_t fptoui_f64_i16(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint16_t) (int16_t) x;+  }+}++SCALAR_FUN_ATTR uint32_t fptoui_f64_i32(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint32_t) (int32_t) x;+  }+}++SCALAR_FUN_ATTR uint64_t fptoui_f64_i64(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint64_t) (int64_t) x;+  }+}++SCALAR_FUN_ATTR bool ftob_f64_bool(double x) {+  return x != 0.0;+}++SCALAR_FUN_ATTR double btof_bool_f64(bool x) {+  return x ? 1.0 : 0.0;+}++SCALAR_FUN_ATTR int64_t futrts_to_bits64(double x) {+  int64_t res;+  foreach_active (i) {+    uniform double tmp = extract(x, i);+    uniform int64_t r = *((uniform int64_t* uniform)&tmp);+    res = insert(res, i, r);+  }+  return res;+}++SCALAR_FUN_ATTR double futrts_from_bits64(int64_t x) {+  double res;+  foreach_active (i) {+    uniform int64_t tmp = extract(x, i);+    uniform double r = *((uniform double* uniform)&tmp);+    res = insert(res, i, r);+  }+  return res;+}++SCALAR_FUN_ATTR double fmod64(double x, double y) {+  return x - y * trunc(x/y);+}++SCALAR_FUN_ATTR double fsignum64(double x) {+  return futrts_isnan64(x) ? x : (x > 0 ? 1.0d : 0.0d) - (x < 0 ? 1.0d : 0.0d);+}++SCALAR_FUN_ATTR double futrts_lerp64(double v0, double v1, double t) {+  return v0 + (v1 - v0) * t;+}++SCALAR_FUN_ATTR double futrts_mad64(double a, double b, double c) {+  return a * b + c;+}++SCALAR_FUN_ATTR float fpconv_f32_f32(float x) {+  return (float) x;+}++SCALAR_FUN_ATTR double fpconv_f32_f64(float x) {+  return (double) x;+}++SCALAR_FUN_ATTR float fpconv_f64_f32(double x) {+  return (float) x;+}++SCALAR_FUN_ATTR double fpconv_f64_f64(double x) {+  return (double) x;+}++#else++SCALAR_FUN_ATTR double fdiv64(double x, double y) {+  return x / y;+}++SCALAR_FUN_ATTR double fadd64(double x, double y) {+  return x + y;+}++SCALAR_FUN_ATTR double fsub64(double x, double y) {+  return x - y;+}++SCALAR_FUN_ATTR double fmul64(double x, double y) {+  return x * y;+}++SCALAR_FUN_ATTR bool cmplt64(double x, double y) {+  return x < y;+}++SCALAR_FUN_ATTR bool cmple64(double x, double y) {+  return x <= y;+}++SCALAR_FUN_ATTR double sitofp_i8_f64(int8_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double sitofp_i16_f64(int16_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double sitofp_i32_f64(int32_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double sitofp_i64_f64(int64_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double uitofp_i8_f64(uint8_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double uitofp_i16_f64(uint16_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double uitofp_i32_f64(uint32_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double uitofp_i64_f64(uint64_t x) {+  return (double) x;+}++SCALAR_FUN_ATTR double fabs64(double x) {+  return fabs(x);+}++SCALAR_FUN_ATTR double fmax64(double x, double y) {+  return fmax(x, y);+}++SCALAR_FUN_ATTR double fmin64(double x, double y) {+  return fmin(x, y);+}++SCALAR_FUN_ATTR double fpow64(double x, double y) {+  return pow(x, y);+}++SCALAR_FUN_ATTR double futrts_log64(double x) {+  return log(x);+}++SCALAR_FUN_ATTR double futrts_log2_64(double x) {+  return log2(x);+}++SCALAR_FUN_ATTR double futrts_log10_64(double x) {+  return log10(x);+}++SCALAR_FUN_ATTR double futrts_log1p_64(double x) {+  return log1p(x);+}++SCALAR_FUN_ATTR double futrts_sqrt64(double x) {+  return sqrt(x);+}++SCALAR_FUN_ATTR double futrts_cbrt64(double x) {+  return cbrt(x);+}++SCALAR_FUN_ATTR double futrts_exp64(double x) {+  return exp(x);+}++SCALAR_FUN_ATTR double futrts_cos64(double x) {+  return cos(x);+}++SCALAR_FUN_ATTR double futrts_sin64(double x) {+  return sin(x);+}++SCALAR_FUN_ATTR double futrts_tan64(double x) {+  return tan(x);+}++SCALAR_FUN_ATTR double futrts_acos64(double x) {+  return acos(x);+}++SCALAR_FUN_ATTR double futrts_asin64(double x) {+  return asin(x);+}++SCALAR_FUN_ATTR double futrts_atan64(double x) {+  return atan(x);+}++SCALAR_FUN_ATTR double futrts_cosh64(double x) {+  return cosh(x);+}++SCALAR_FUN_ATTR double futrts_sinh64(double x) {+  return sinh(x);+}++SCALAR_FUN_ATTR double futrts_tanh64(double x) {+  return tanh(x);+}++SCALAR_FUN_ATTR double futrts_acosh64(double x) {+  return acosh(x);+}++SCALAR_FUN_ATTR double futrts_asinh64(double x) {+  return asinh(x);+}++SCALAR_FUN_ATTR double futrts_atanh64(double x) {+  return atanh(x);+}++SCALAR_FUN_ATTR double futrts_atan2_64(double x, double y) {+  return atan2(x, y);+}++SCALAR_FUN_ATTR double futrts_hypot64(double x, double y) {+  return hypot(x, y);+}++SCALAR_FUN_ATTR double futrts_gamma64(double x) {+  return tgamma(x);+}++SCALAR_FUN_ATTR double futrts_lgamma64(double x) {+  return lgamma(x);+}++SCALAR_FUN_ATTR double futrts_erf64(double x) {+  return erf(x);+}++SCALAR_FUN_ATTR double futrts_erfc64(double x) {+  return erfc(x);+}++SCALAR_FUN_ATTR double futrts_fma64(double a, double b, double c) {+  return fma(a, b, c);+}++SCALAR_FUN_ATTR double futrts_round64(double x) {+  return rint(x);+}++SCALAR_FUN_ATTR double futrts_ceil64(double x) {+  return ceil(x);+}++SCALAR_FUN_ATTR float futrts_nextafter64(float x, float y) {+  return nextafter(x, y);+}++SCALAR_FUN_ATTR double futrts_floor64(double x) {+  return floor(x);+}++SCALAR_FUN_ATTR bool futrts_isnan64(double x) {+  return isnan(x);+}++SCALAR_FUN_ATTR bool futrts_isinf64(double x) {+  return isinf(x);+}++SCALAR_FUN_ATTR int8_t fptosi_f64_i8(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int8_t) x;+  }+}++SCALAR_FUN_ATTR int16_t fptosi_f64_i16(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int16_t) x;+  }+}++SCALAR_FUN_ATTR int32_t fptosi_f64_i32(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int32_t) x;+  }+}++SCALAR_FUN_ATTR int64_t fptosi_f64_i64(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int64_t) x;+  }+}++SCALAR_FUN_ATTR uint8_t fptoui_f64_i8(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint8_t) (int8_t) x;+  }+}++SCALAR_FUN_ATTR uint16_t fptoui_f64_i16(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint16_t) (int16_t) x;+  }+}++SCALAR_FUN_ATTR uint32_t fptoui_f64_i32(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint32_t) (int32_t) x;+  }+}++SCALAR_FUN_ATTR uint64_t fptoui_f64_i64(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint64_t) (int64_t) x;+  }+}++SCALAR_FUN_ATTR bool ftob_f64_bool(double x) {+  return x != 0;+}++SCALAR_FUN_ATTR double btof_bool_f64(bool x) {+  return x ? 1 : 0;+}++SCALAR_FUN_ATTR int64_t futrts_to_bits64(double x) {+  union {+    double f;+    int64_t t;+  } p;++  p.f = x;+  return p.t;+}++SCALAR_FUN_ATTR double futrts_from_bits64(int64_t x) {+  union {+    int64_t f;+    double t;+  } p;++  p.f = x;+  return p.t;+}++SCALAR_FUN_ATTR double fmod64(double x, double y) {+  return fmod(x, y);+}++SCALAR_FUN_ATTR double fsignum64(double x) {+  return futrts_isnan64(x) ? x : (x > 0) - (x < 0);+}++SCALAR_FUN_ATTR double futrts_lerp64(double v0, double v1, double t) {+#ifdef __OPENCL_VERSION__+  return mix(v0, v1, t);+#else+  return v0 + (v1 - v0) * t;+#endif+}++SCALAR_FUN_ATTR double futrts_mad64(double a, double b, double c) {+#ifdef __OPENCL_VERSION__+  return mad(a, b, c);+#else+  return a * b + c;+#endif+}++SCALAR_FUN_ATTR float fpconv_f32_f32(float x) {+  return (float) x;+}++SCALAR_FUN_ATTR double fpconv_f32_f64(float x) {+  return (double) x;+}++SCALAR_FUN_ATTR float fpconv_f64_f32(double x) {+  return (float) x;+}++SCALAR_FUN_ATTR double fpconv_f64_f64(double x) {   return (double) x; }