futhark 0.21.12 → 0.21.13

raw patch · 233 files changed

+17048/−9768 lines, 233 filesdep +co-log-coredep +fgldep +fgl-visualizedep −hsloggerdep −parser-combinatorsdep ~futhark-manifestdep ~futhark-serverdep ~lspPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: co-log-core, fgl, fgl-visualize

Dependencies removed: hslogger, parser-combinators

Dependency ranges changed: futhark-manifest, futhark-server, lsp, mtl

API changes (from Hackage documentation)

Files

• LICENSE +1/−1
• docs/c-api.rst +56/−7
• docs/conf.py +1/−0
• docs/index.rst +1/−0
• docs/installation.rst +1/−1
• docs/language-reference.rst +2/−2
• docs/man/futhark-ispc.rst +61/−0
• docs/man/futhark-literate.rst +6/−0
• docs/man/futhark.rst +6/−1
• docs/server-protocol.rst +40/−1
• futhark.cabal +67/−31
• prelude/soacs.fut +40/−36
• rts/c/ispc_util.h +457/−0
• rts/c/scalar.h +2923/−1699
• rts/c/scalar_f16.h +198/−6
• rts/c/scheduler.h +15/−5
• rts/c/server.h +190/−32
• rts/c/uniform.h +1708/−0
• rts/c/util.h +5/−0
• rts/python/server.py +11/−1
• src/Futhark/AD/Derivatives.hs +1/−1
• src/Futhark/AD/Fwd.hs +6/−6
• src/Futhark/AD/Rev.hs +10/−9
• src/Futhark/AD/Rev/Loop.hs +34/−33
• src/Futhark/AD/Rev/Map.hs +4/−4
• src/Futhark/AD/Rev/Monad.hs +14/−14
• src/Futhark/AD/Rev/Reduce.hs +7/−4
• src/Futhark/AD/Rev/Scan.hs +3/−1
• src/Futhark/AD/Rev/Scatter.hs +2/−1
• src/Futhark/Actions.hs +84/−2
• src/Futhark/Analysis/Alias.hs +7/−3
• src/Futhark/Analysis/CallGraph.hs +2/−35
• src/Futhark/Analysis/HORep/MapNest.hs +2/−1
• src/Futhark/Analysis/HORep/SOAC.hs +39/−26
• src/Futhark/Analysis/Interference.hs +6/−4
• src/Futhark/Analysis/LastUse.hs +19/−9
• src/Futhark/Analysis/MemAlias.hs +7/−12
• src/Futhark/Analysis/PrimExp.hs +2/−2
• src/Futhark/Analysis/PrimExp/Convert.hs +3/−1
• src/Futhark/Analysis/PrimExp/Parse.hs +1/−1
• src/Futhark/Analysis/Rephrase.hs +4/−4
• src/Futhark/Analysis/SymbolTable.hs +2/−1
• src/Futhark/Analysis/UsageTable.hs +5/−5
• src/Futhark/Bench.hs +3/−1
• src/Futhark/Builder.hs +7/−1
• src/Futhark/CLI/Autotune.hs +20/−13
• src/Futhark/CLI/Bench.hs +4/−2
• src/Futhark/CLI/Check.hs +3/−1
• src/Futhark/CLI/Dataset.hs +2/−2
• src/Futhark/CLI/Dev.hs +16/−5
• src/Futhark/CLI/Doc.hs +2/−1
• src/Futhark/CLI/LSP.hs +0/−5
• src/Futhark/CLI/Literate.hs +53/−27
• src/Futhark/CLI/Main.hs +5/−1
• src/Futhark/CLI/Misc.hs +27/−1
• src/Futhark/CLI/MulticoreISPC.hs +19/−0
• src/Futhark/CLI/Pkg.hs +14/−5
• src/Futhark/CLI/REPL.hs +2/−1
• src/Futhark/CLI/Run.hs +6/−2
• src/Futhark/CLI/Test.hs +32/−14
• src/Futhark/CodeGen/Backends/CCUDA.hs +21/−18
• src/Futhark/CodeGen/Backends/CCUDA/Boilerplate.hs +2/−2
• src/Futhark/CodeGen/Backends/COpenCL.hs +17/−11
• src/Futhark/CodeGen/Backends/GenericC.hs +411/−166
• src/Futhark/CodeGen/Backends/GenericC/CLI.hs +5/−4
• src/Futhark/CodeGen/Backends/GenericC/Options.hs +5/−5
• src/Futhark/CodeGen/Backends/GenericC/Server.hs +73/−29
• src/Futhark/CodeGen/Backends/GenericPython.hs +106/−94
• src/Futhark/CodeGen/Backends/GenericPython/AST.hs +23/−10
• src/Futhark/CodeGen/Backends/GenericPython/Options.hs +3/−1
• src/Futhark/CodeGen/Backends/GenericWASM.hs +20/−18
• src/Futhark/CodeGen/Backends/MulticoreC.hs +47/−7
• src/Futhark/CodeGen/Backends/MulticoreISPC.hs +1114/−0
• src/Futhark/CodeGen/Backends/MulticoreWASM.hs +5/−5
• src/Futhark/CodeGen/Backends/PyOpenCL.hs +7/−5
• src/Futhark/CodeGen/Backends/PyOpenCL/Boilerplate.hs +2/−1
• src/Futhark/CodeGen/Backends/SequentialC.hs +1/−1
• src/Futhark/CodeGen/Backends/SequentialWASM.hs +4/−4
• src/Futhark/CodeGen/Backends/SimpleRep.hs +31/−33
• src/Futhark/CodeGen/ImpCode.hs +120/−93
• src/Futhark/CodeGen/ImpCode/GPU.hs +79/−34
• src/Futhark/CodeGen/ImpCode/Multicore.hs +51/−2
• src/Futhark/CodeGen/ImpGen.hs +136/−84
• src/Futhark/CodeGen/ImpGen/GPU.hs +9/−8
• src/Futhark/CodeGen/ImpGen/GPU/Base.hs +89/−55
• src/Futhark/CodeGen/ImpGen/GPU/SegHist.hs +66/−39
• src/Futhark/CodeGen/ImpGen/GPU/SegMap.hs +3/−3
• src/Futhark/CodeGen/ImpGen/GPU/SegRed.hs +53/−42
• src/Futhark/CodeGen/ImpGen/GPU/SegScan/SinglePass.hs +46/−36
• src/Futhark/CodeGen/ImpGen/GPU/SegScan/TwoPass.hs +18/−12
• src/Futhark/CodeGen/ImpGen/GPU/ToOpenCL.hs +9/−5
• src/Futhark/CodeGen/ImpGen/GPU/Transpose.hs +8/−4
• src/Futhark/CodeGen/ImpGen/Multicore.hs +5/−12
• src/Futhark/CodeGen/ImpGen/Multicore/Base.hs +115/−9
• src/Futhark/CodeGen/ImpGen/Multicore/SegHist.hs +82/−42
• src/Futhark/CodeGen/ImpGen/Multicore/SegMap.hs +5/−4
• src/Futhark/CodeGen/ImpGen/Multicore/SegRed.hs +265/−96
• src/Futhark/CodeGen/ImpGen/Multicore/SegScan.hs +225/−96
• src/Futhark/CodeGen/ImpGen/Transpose.hs +0/−2
• src/Futhark/CodeGen/RTS/C.hs +12/−0
• src/Futhark/CodeGen/SetDefaultSpace.hs +14/−8
• src/Futhark/Compiler.hs +5/−4
• src/Futhark/Compiler/Program.hs +6/−2
• src/Futhark/Construct.hs +9/−4
• src/Futhark/Doc/Generator.hs +52/−42
• src/Futhark/IR/Aliases.hs +3/−1
• src/Futhark/IR/GPU/Op.hs +4/−2
• src/Futhark/IR/GPU/Simplify.hs +33/−3
• src/Futhark/IR/MC/Op.hs +4/−2
• src/Futhark/IR/Mem.hs +34/−16
• src/Futhark/IR/Mem/IxFun.hs +9/−7
• src/Futhark/IR/Mem/Simplify.hs +3/−1
• src/Futhark/IR/Parse.hs +220/−96
• src/Futhark/IR/Pretty.hs +52/−19
• src/Futhark/IR/Primitive.hs +0/−1873
• src/Futhark/IR/Primitive/Parse.hs +0/−112
• src/Futhark/IR/Prop.hs +9/−16
• src/Futhark/IR/Prop/Constants.hs +2/−1
• src/Futhark/IR/Prop/Names.hs +2/−1
• src/Futhark/IR/Prop/Scope.hs +2/−1
• src/Futhark/IR/Prop/Types.hs +7/−7
• src/Futhark/IR/Rep.hs +1/−1
• src/Futhark/IR/RetType.hs +2/−1
• src/Futhark/IR/SOACS/SOAC.hs +114/−49
• src/Futhark/IR/SOACS/Simplify.hs +40/−16
• src/Futhark/IR/SegOp.hs +47/−21
• src/Futhark/IR/Syntax.hs +18/−20
• src/Futhark/IR/Syntax/Core.hs +54/−2
• src/Futhark/IR/Traversals.hs +23/−9
• src/Futhark/IR/TypeCheck.hs +214/−183
• src/Futhark/Internalise.hs +2/−1
• src/Futhark/Internalise/Bindings.hs +7/−5
• src/Futhark/Internalise/Defunctionalise.hs +70/−57
• src/Futhark/Internalise/Defunctorise.hs +20/−8
• src/Futhark/Internalise/Entry.hs +165/−0
• src/Futhark/Internalise/Exps.hs +172/−184
• src/Futhark/Internalise/FreeVars.hs +0/−132
• src/Futhark/Internalise/Lambdas.hs +4/−20
• src/Futhark/Internalise/LiftLambdas.hs +14/−12
• src/Futhark/Internalise/Monad.hs +47/−25
• src/Futhark/Internalise/Monomorphise.hs +62/−45
• src/Futhark/Internalise/TypesValues.hs +49/−41
• src/Futhark/LSP/Compile.hs +12/−11
• src/Futhark/LSP/Diagnostic.hs +3/−2
• src/Futhark/LSP/Handlers.hs +12/−12
• src/Futhark/Optimise/BlkRegTiling.hs +19/−13
• src/Futhark/Optimise/CSE.hs +4/−1
• src/Futhark/Optimise/DoubleBuffer.hs +9/−5
• src/Futhark/Optimise/Fusion.hs +479/−1133
• src/Futhark/Optimise/Fusion/Composing.hs +6/−3
• src/Futhark/Optimise/Fusion/GraphRep.hs +431/−0
• src/Futhark/Optimise/Fusion/LoopKernel.hs +0/−937
• src/Futhark/Optimise/Fusion/TryFusion.hs +856/−0
• src/Futhark/Optimise/GenRedOpt.hs +8/−6
• src/Futhark/Optimise/HistAccs.hs +6/−2
• src/Futhark/Optimise/InPlaceLowering.hs +5/−2
• src/Futhark/Optimise/InPlaceLowering/LowerIntoStm.hs +14/−9
• src/Futhark/Optimise/InPlaceLowering/SubstituteIndices.hs +19/−7
• src/Futhark/Optimise/InliningDeadFun.hs +10/−5
• src/Futhark/Optimise/ReduceDeviceSyncs.hs +37/−44
• src/Futhark/Optimise/ReduceDeviceSyncs/MigrationTable.hs +6/−15
• src/Futhark/Optimise/Simplify.hs +8/−2
• src/Futhark/Optimise/Simplify/Engine.hs +36/−30
• src/Futhark/Optimise/Simplify/Rules.hs +31/−14
• src/Futhark/Optimise/Simplify/Rules/BasicOp.hs +50/−36
• src/Futhark/Optimise/Simplify/Rules/ClosedForm.hs +6/−3
• src/Futhark/Optimise/Simplify/Rules/Index.hs +6/−3
• src/Futhark/Optimise/Simplify/Rules/Loop.hs +9/−6
• src/Futhark/Optimise/Simplify/Rules/Simple.hs +14/−0
• src/Futhark/Optimise/Sink.hs +6/−3
• src/Futhark/Optimise/TileLoops.hs +36/−21
• src/Futhark/Optimise/TileLoops/Shared.hs +14/−4
• src/Futhark/Optimise/Unstream.hs +4/−2
• src/Futhark/Pass.hs +4/−19
• src/Futhark/Pass/ExpandAllocations.hs +27/−15
• src/Futhark/Pass/ExplicitAllocations.hs +17/−6
• src/Futhark/Pass/ExplicitAllocations/GPU.hs +4/−2
• src/Futhark/Pass/ExplicitAllocations/SegOp.hs +5/−2
• src/Futhark/Pass/ExtractKernels.hs +46/−26
• src/Futhark/Pass/ExtractKernels/BlockedKernel.hs +3/−1
• src/Futhark/Pass/ExtractKernels/DistributeNests.hs +44/−20
• src/Futhark/Pass/ExtractKernels/Distribution.hs +5/−3
• src/Futhark/Pass/ExtractKernels/ISRWIM.hs +21/−10
• src/Futhark/Pass/ExtractKernels/Interchange.hs +9/−5
• src/Futhark/Pass/ExtractKernels/Intragroup.hs +7/−3
• src/Futhark/Pass/ExtractKernels/StreamKernel.hs +5/−2
• src/Futhark/Pass/ExtractMulticore.hs +17/−8
• src/Futhark/Pass/KernelBabysitting.hs +13/−7
• src/Futhark/Passes.hs +0/−6
• src/Futhark/Pkg/Info.hs +18/−6
• src/Futhark/Pkg/Solve.hs +4/−1
• src/Futhark/Pkg/Types.hs +9/−4
• src/Futhark/Script.hs +82/−34
• src/Futhark/Test.hs +13/−8
• src/Futhark/Test/Spec.hs +5/−2
• src/Futhark/Tools.hs +2/−1
• src/Futhark/Transform/FirstOrderTransform.hs +23/−11
• src/Futhark/Util.hs +4/−11
• src/Language/Futhark.hs +3/−1
• src/Language/Futhark/Core.hs +2/−9
• src/Language/Futhark/FreeVars.hs +164/−0
• src/Language/Futhark/Interpreter.hs +69/−49
• src/Language/Futhark/Parser/Monad.hs +6/−3
• src/Language/Futhark/Parser/Parser.y +13/−13
• src/Language/Futhark/Pretty.hs +64/−43
• src/Language/Futhark/Primitive.hs +1873/−0
• src/Language/Futhark/Primitive/Parse.hs +111/−0
• src/Language/Futhark/Prop.hs +127/−199
• src/Language/Futhark/Query.hs +3/−2
• src/Language/Futhark/Semantic.hs +10/−4
• src/Language/Futhark/Syntax.hs +77/−102
• src/Language/Futhark/Traversals.hs +61/−38
• src/Language/Futhark/Tuple.hs +43/−0
• src/Language/Futhark/TypeChecker.hs +13/−9
• src/Language/Futhark/TypeChecker/Modules.hs +28/−21
• src/Language/Futhark/TypeChecker/Monad.hs +20/−17
• src/Language/Futhark/TypeChecker/Terms.hs +153/−137
• src/Language/Futhark/TypeChecker/Terms/DoLoop.hs +19/−17
• src/Language/Futhark/TypeChecker/Terms/Monad.hs +84/−52
• src/Language/Futhark/TypeChecker/Terms/Pat.hs +19/−9
• src/Language/Futhark/TypeChecker/Types.hs +141/−68
• src/Language/Futhark/TypeChecker/Unify.hs +121/−88
• unittests/Futhark/AD/DerivativesTests.hs +2/−1
• unittests/Futhark/BenchTests.hs +3/−1
• unittests/Futhark/IR/Mem/IxFun/Alg.hs +2/−1
• unittests/Futhark/IR/Mem/IxFunTests.hs +3/−1
• unittests/Futhark/IR/PrimitiveTests.hs +0/−77
• unittests/Futhark/IR/Syntax/CoreTests.hs +4/−2
• unittests/Futhark/Optimise/MemoryBlockMerging/GreedyColoringTests.hs +17/−15
• unittests/Language/Futhark/CoreTests.hs +1/−1
• unittests/Language/Futhark/PrimitiveTests.hs +77/−0
• unittests/Language/Futhark/SyntaxTests.hs +14/−16
• unittests/futhark_tests.hs +2/−2

@@ -1,6 +1,6 @@ ISC License -Copyright (c) 2013-2018. DIKU, University of Copenhagen+Copyright (c) 2013-2022. DIKU, University of Copenhagen  Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the

@@ -332,19 +332,68 @@     * Otherwise, the serialised representation of the value will be      stored at ``*p``, which *must* have room for at least ``*n``-     bytes.+     bytes.  This is done asynchronously.     Returns 0 on success.  .. c:function:: struct futhark_opaque_foo* futhark_restore_opaque_foo(struct futhark_context *ctx, const void *p) -   Restore a byte sequence previously written with+   Asynchronously restore a byte sequence previously written with    :c:func:`futhark_store_opaque_foo`.  Returns ``NULL`` on failure.    The byte sequence does not need to have been generated by the same    program *instance*, but it *must* have been generated by the same    Futhark program, and compiled with the same version of the Futhark    compiler. +Records+~~~~~~~++A record is an opaque type (see above) that supports additional+functions to *project* individual fields (read their values) to+construct a value given values for the fields.  An opaque type is a+record if its definition is a record at the Futhark level.++The projection and construction functions are equivalent in+functionality to writing entry points by hand, and so serve only to+cut down on boilerplate.  Important things to be aware of:++1. The objects constructed though these functions have their own+   lifetime (like any objects returned from an entry point) and must+   be manually freed, independently of the records from which they are+   projected, or the fields they are constructed from.++2. The objects are however in an *aliasing* relationship with the+   fields or original record.  This means you must be careful when+   passing them to entry points that consume their arguments.  As+   always, you don't have to worry about this if you never write entry+   points that consume their arguments.++The precise functions generated depend on the fields of the record.+The following functions assume a record with Futhark-level type ``type+t = {foo: t1, bar: t2}`` where ``t1`` and ``t2`` are also opaque+types.++.. c:function:: int futhark_new_opaque_t(struct futhark_context *ctx, struct futhark_opaque_t **out, const struct futhark_opaque_t2 *bar, const struct futhark_opaque_t1 *foo);++   Construct a record in ``*out`` which has the given values for the+   ``bar`` and ``foo`` fields.  The parameter ordering constitutes the+   fields in alphabetic order.  Tuple fields are named ``vX`` where+   ``X`` is an integer.  The resulting record *aliases* the values+   provided for ``bar`` and ``foo``, but has its own lifetime, and all+   values must be individually freed when they are no longer needed.++.. c:function:: int futhark_project_opaque_t_bar(struct futhark_context *ctx, struct futhark_opaque_t2 **out, const struct futhark_opaque_t *obj);++   Extract the value of the field ``bar`` from the provided record.+   The resulting value *aliases* the record, but has its own lifetime,+   and must eventually be freed.++.. c:function:: int futhark_project_opaque_t_foo(struct futhark_context *ctx, struct futhark_opaque_t1 **out, const struct futhark_opaque_t *obj);++   Extract the value of the field ``bar`` from the provided record.+   The resulting value *aliases* the record, but has its own lifetime,+   and must eventually be freed.+ Entry points ------------ @@ -507,7 +556,7 @@ The following API functions are available when using the ``multicore`` backend. -.. c:function:: void context_config_set_num_threads(struct futhark_context_config *cfg, int n)+.. c:function:: void futhark_context_config_set_num_threads(struct futhark_context_config *cfg, int n)     The number of threads used to run parallel operations.  If set to a    value less than ``1``, then the runtime system will use one thread@@ -564,15 +613,15 @@   * A list of all *outputs*, including their type and whether they are     *unique*. -* A mapping from the name of each non-scalar types to:+* A mapping from the name of each non-scalar type to: -  * The C type of used to represent type type (which is practice+  * The C type of used to represent the type (which is in practice     always a pointer of some kind).    * For arrays, the element type and rank. -  * A mapping from names of *operations* to the name of the C function-    that implements that operation for the type.  The type of the C+  * A mapping from *operations* to the names of the C functions that+    implement the operations for the type.  The types of the C     functions are as documented above.  The following operations are     listed: 

@@ -343,6 +343,7 @@     ('man/futhark-autotune', 'futhark-autotune', 'calibrate run-time parameters', [], 1),     ('man/futhark-c', 'futhark-c', 'compile Futhark to sequential C', [], 1),     ('man/futhark-multicore', 'futhark-multicore', 'compile Futhark to multithreaded C', [], 1),+    ('man/futhark-ispc', 'futhark-ispc', 'compile Futhark to multithreaded ISPC', [], 1),     ('man/futhark-opencl', 'futhark-opencl', 'compile Futhark to OpenCL', [], 1),     ('man/futhark-cuda', 'futhark-cuda', 'compile Futhark to CUDA', [], 1),     ('man/futhark-python', 'futhark-python', 'compile Futhark to sequential Python', [], 1),

@@ -53,6 +53,7 @@    man/futhark-doc.rst    man/futhark-literate.rst    man/futhark-multicore.rst+   man/futhark-ispc.rst    man/futhark-opencl.rst    man/futhark-pkg.rst    man/futhark-pyopencl.rst

@@ -37,7 +37,7 @@ ---------------------  To compile Futhark you must first install an appropriate version of-GHC, either with [ghcup](https://www.haskell.org/ghcup/) or a package+GHC, either with `ghcup <https://www.haskell.org/ghcup>`_ or a package manager.  Any version since GHC 8.10 should work.  You also need the ``cabal`` command line program, which ghcup will install for you as well.

@@ -396,8 +396,8 @@  A type abbreviation can have zero or more parameters.  A type parameter enclosed with square brackets is a *size parameter*, and-can be used in the definition as an array dimension size, or as a-dimension argument to other type abbreviations.  When passing an+can be used in the definition as an array size, or as a+size argument to other type abbreviations.  When passing an argument for a shape parameter, it must be enclosed in square brackets.  Example:: 

@@ -0,0 +1,61 @@+.. role:: ref(emphasis)++.. _futhark-ispc(1):++============+futhark-ispc+============++SYNOPSIS+========++futhark ispc [options...] <program.fut>++DESCRIPTION+===========++``futhark ispc`` translates a Futhark program to a combination of C+and ISPC code, with ISPC used for parallel loops.  It otherwise+operates similarly to :ref:`futhark-multicore(1)`.  You need to have+``ispc`` on your ``PATH``.++OPTIONS+=======++Accepts the same options as :ref:`futhark-multicore(1)`.+++ENVIRONMENT VARIABLES+=====================++``CC``++  The C compiler used to compile the program.  Defaults to ``cc`` if+  unset.++``CFLAGS``++  Space-separated list of options passed to the C compiler.  Defaults+  to ``-O3 -std=c99 -pthread`` if unset.++``ISPCFLAGS``++  Space-separated list of options passed to ``ispc``.  Defaults to+  ``-O3 --woff`` if unset.++EXECUTABLE OPTIONS+==================++Generated executables accept the same options as those generated by+:ref:`futhark-multicore(1)`.++BUGS+====++Currently works only on Unix-like systems because of a dependency on+pthreads.  Adding support for Windows would likely not be difficult.++SEE ALSO+========++:ref:`futhark-multicore(1)`, :ref:`futhark-test(1)`

@@ -239,6 +239,12 @@ running a program from an unknown source, you should always give it a quick read to see if anything looks fishy. +BUGS+====++FutharkScript expressions can only refer to names defined in the file+passed to ``futhark literate``, not any names in imported files.+ SEE ALSO ======== 

@@ -95,7 +95,12 @@  Expresses gratitude. +futhark tokens FILE+-------------------++Print the tokens the given Futhark source file; one per line.+ SEE ALSO ======== -:ref:`futhark-opencl(1)`, :ref:`futhark-c(1)`, :ref:`futhark-py(1)`, :ref:`futhark-pyopencl(1)`, :ref:`futhark-wasm(1)`, :ref:`futhark-wasm-multicore(1)`, :ref:`futhark-dataset(1)`, :ref:`futhark-doc(1)`, :ref:`futhark-test(1)`, :ref:`futhark-bench(1)`, :ref:`futhark-run(1)`, :ref:`futhark-repl(1)`, :ref:`futhark-literate(1)`+:ref:`futhark-opencl(1)`, :ref:`futhark-c(1)`, :ref:`futhark-py(1)`, :ref:`futhark-pyopencl(1)`, :ref:`futhark-wasm(1)`, :ref:`futhark-wasm-multicore(1)`, :ref:`futhark-ispc(1)`, :ref:`futhark-dataset(1)`, :ref:`futhark-doc(1)`, :ref:`futhark-test(1)`, :ref:`futhark-bench(1)`, :ref:`futhark-run(1)`, :ref:`futhark-repl(1)`, :ref:`futhark-literate(1)`

@@ -71,6 +71,19 @@  The following commands are supported. +General Commands+~~~~~~~~~~~~~~~~++``types``+.........++Print the names of available types, one per line.++``entry_points``+................++Print the names of available entry points.+ ``call`` *entry* *o1* ... *oN* *i1* ... *iM* ............................................ @@ -138,10 +151,36 @@  Corresponds to :c:func:`futhark_context_config_set_tuning_param`. +Record Commands+~~~~~~~~~~~~~~~++``fields`` *type*+.................++If the given type is a record, print a line for each field of the+record.  The line will contain the name of the field, followed by a+space, followed by the type of the field.  Note that the type name can+contain spaces.  The order of fields is significant, as it is the one+expected by the ``new_record`` command.++``new`` *v0* *type* *v1* ... *vN*+.................................++Create a new variable *v0* of type *type*, which must be a record type+with *N* fields, where *v1* to *vN* are variables with the+corresponding field types (the expected order is given by the+``fields`` command).++``project`` *to* *from* *field*+...............................++Create a new variable *to* whose value is the field *field* of the+record-typed variable *from*.+ Environment Variables ---------------------  ``FUTHARK_COMPILER_DEBUGGING``-..............................+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~  Turns on debugging output for the server when set to 1.

@@ -1,6 +1,6 @@ cabal-version: 2.4 name:           futhark-version:        0.21.12+version:        0.21.13 synopsis:       An optimising compiler for a functional, array-oriented language.  description:    Futhark is a small programming language designed to be compiled to@@ -34,11 +34,44 @@ license-file:   LICENSE build-type:     Simple extra-source-files:-    rts/c/*.h-    rts/futhark-doc/*.css-    rts/javascript/*.js-    rts/python/*.py-    prelude/*.fut+-- Cabal's recompilation tracking doesn't work when we use wildcards+-- here, so for now we spell out every single file.+    rts/c/atomics.h+    rts/c/lock.h+    rts/c/timing.h+    rts/c/errors.h+    rts/c/free_list.h+    rts/c/tuning.h+    rts/c/values.h+    rts/c/half.h+    rts/c/opencl.h+    rts/c/cuda.h+    rts/c/cache.h+    rts/c/ispc_util.h+    rts/c/scalar.h+    rts/c/scalar_f16.h+    rts/c/scheduler.h+    rts/c/uniform.h+    rts/c/util.h+    rts/c/server.h+    rts/futhark-doc/style.css+    rts/javascript/server.js+    rts/javascript/values.js+    rts/javascript/wrapperclasses.js+    rts/python/tuning.py+    rts/python/panic.py+    rts/python/memory.py+    rts/python/server.py+    rts/python/values.py+    rts/python/opencl.py+    rts/python/scalar.py+    prelude/functional.fut+    prelude/math.fut+    prelude/soacs.fut+    prelude/zip.fut+    prelude/ad.fut+    prelude/array.fut+    prelude/prelude.fut -- Just enough of the docs to build the manpages.     docs/**/*.rst     docs/Makefile@@ -51,7 +84,13 @@   type: git   location: https://github.com/diku-dk/futhark +common common+  ghc-options: -Wall -Wcompat -Wno-incomplete-uni-patterns -Wredundant-constraints -Wincomplete-record-updates -Wmissing-export-lists -Wunused-packages+  default-language: Haskell2010+ library+  import: common+  hs-source-dirs: src   exposed-modules:       Futhark       Futhark.Actions@@ -101,6 +140,7 @@       Futhark.CLI.Main       Futhark.CLI.Misc       Futhark.CLI.Multicore+      Futhark.CLI.MulticoreISPC       Futhark.CLI.MulticoreWASM       Futhark.CLI.OpenCL       Futhark.CLI.Pkg@@ -124,6 +164,7 @@       Futhark.CodeGen.Backends.GenericPython.Options       Futhark.CodeGen.Backends.GenericWASM       Futhark.CodeGen.Backends.MulticoreC+      Futhark.CodeGen.Backends.MulticoreISPC       Futhark.CodeGen.Backends.MulticoreWASM       Futhark.CodeGen.Backends.PyOpenCL       Futhark.CodeGen.Backends.PyOpenCL.Boilerplate@@ -186,8 +227,6 @@       Futhark.IR.Mem.Simplify       Futhark.IR.Parse       Futhark.IR.Pretty-      Futhark.IR.Primitive-      Futhark.IR.Primitive.Parse       Futhark.IR.Prop       Futhark.IR.Prop.Aliases       Futhark.IR.Prop.Constants@@ -215,8 +254,8 @@       Futhark.Internalise.Bindings       Futhark.Internalise.Defunctionalise       Futhark.Internalise.Defunctorise+      Futhark.Internalise.Entry       Futhark.Internalise.Exps-      Futhark.Internalise.FreeVars       Futhark.Internalise.Lambdas       Futhark.Internalise.LiftLambdas       Futhark.Internalise.Monad@@ -234,7 +273,8 @@       Futhark.Optimise.DoubleBuffer       Futhark.Optimise.Fusion       Futhark.Optimise.Fusion.Composing-      Futhark.Optimise.Fusion.LoopKernel+      Futhark.Optimise.Fusion.GraphRep+      Futhark.Optimise.Fusion.TryFusion       Futhark.Optimise.GenRedOpt       Futhark.Optimise.HistAccs       Futhark.Optimise.InPlaceLowering@@ -309,17 +349,21 @@       Language.Futhark       Language.Futhark.Core       Language.Futhark.Interpreter+      Language.Futhark.FreeVars       Language.Futhark.Parser       Language.Futhark.Parser.Monad       Language.Futhark.Parser.Lexer.Tokens       Language.Futhark.Parser.Lexer.Wrapper       Language.Futhark.Prelude       Language.Futhark.Pretty+      Language.Futhark.Primitive+      Language.Futhark.Primitive.Parse       Language.Futhark.Prop       Language.Futhark.Query       Language.Futhark.Semantic       Language.Futhark.Syntax       Language.Futhark.Traversals+      Language.Futhark.Tuple       Language.Futhark.TypeChecker       Language.Futhark.TypeChecker.Match       Language.Futhark.TypeChecker.Modules@@ -337,9 +381,6 @@       Paths_futhark   autogen-modules:       Paths_futhark-  hs-source-dirs:-      src-  ghc-options: -Wall -Wcompat -Wno-incomplete-uni-patterns -Wredundant-constraints -Wincomplete-record-updates -Wmissing-export-lists -Wunused-packages   build-tool-depends:       alex:alex     , happy:happy@@ -354,25 +395,27 @@     , bytestring >=0.10.8     , bytestring-to-vector >=0.3.0.1     , bmp >=1.2.6.3+    , co-log-core     , containers >=0.6.2.1     , cryptohash-md5     , Diff >=0.4.1     , directory >=1.3.0.0     , directory-tree >=0.12.1     , dlist >=0.6.0.1+    , fgl+    , fgl-visualize     , file-embed >=0.0.14.0     , filepath >=1.4.1.1     , free >=4.12.4     , futhark-data >= 1.1.0.0-    , futhark-server >= 1.2.0.0-    , futhark-manifest >= 1.0.0.0+    , futhark-server >= 1.2.1.0+    , futhark-manifest >= 1.1.0.0     , githash >=0.1.6.1     , half >= 0.3     , haskeline-    , hslogger-    , language-c-quote >=0.12+    , language-c-quote >= 0.12     , lens-    , lsp >= 1.4+    , lsp == 1.5.*     , mainland-pretty >=0.7.1     , cmark-gfm >=0.2.1     , megaparsec >=9.0.0@@ -396,19 +439,18 @@     , zlib >=0.6.1.2     , statistics     , mwc-random-  default-language: Haskell2010  executable futhark+  import: common   main-is: src/main.hs-  ghc-options: -Wall -Wcompat -Wno-incomplete-uni-patterns -Wredundant-constraints -Wincomplete-record-updates -Wmissing-export-lists -Wunused-packages -threaded -rtsopts "-with-rtsopts=-N -qg1 -A16M"-  build-depends:-      base-    , futhark-  default-language: Haskell2010+  ghc-options: -threaded -rtsopts "-with-rtsopts=-N -qg1 -A16M"+  build-depends: base, futhark  test-suite unit+  import: common   type: exitcode-stdio-1.0   main-is: futhark_tests.hs+  hs-source-dirs: unittests   other-modules:       Futhark.AD.DerivativesTests       Futhark.BenchTests@@ -421,26 +463,20 @@       Futhark.IR.Mem.IxFun.Alg       Futhark.IR.Mem.IxFunTests       Futhark.IR.Mem.IxFunWrapper-      Futhark.IR.PrimitiveTests       Language.Futhark.CoreTests+      Language.Futhark.PrimitiveTests       Language.Futhark.SyntaxTests       Language.Futhark.TypeCheckerTests       Language.Futhark.TypeChecker.TypesTests       Futhark.Optimise.MemoryBlockMerging.GreedyColoringTests       Paths_futhark-  hs-source-dirs:-      unittests-  ghc-options: -Wall -Wcompat -Wredundant-constraints -Wincomplete-record-updates -Wmissing-export-lists   build-depends:       QuickCheck >=2.8     , base     , containers     , futhark     , megaparsec-    , mtl-    , parser-combinators     , tasty     , tasty-hunit     , tasty-quickcheck     , text-  default-language: Haskell2010

@@ -1,7 +1,7 @@ -- | Various Second-Order Array Combinators that are operationally -- parallel in a way that can be exploited by the compiler. ----- The functions here are all recognised specially by the compiler (or+-- The functions here are recognised specially by the compiler (or -- built on those that are).  The asymptotic [work and -- span](https://en.wikipedia.org/wiki/Analysis_of_parallel_algorithms) -- is provided for each function, but note that this easily hides very@@ -117,14 +117,26 @@ def reduce_comm [n] 'a (op: a -> a -> a) (ne: a) (as: [n]a): a =   intrinsics.reduce_comm (op, ne, as) --- | `reduce_by_index dest f ne is as` returns `dest`, but with each--- element given by the indices of `is` updated by applying `f` to the--- current value in `dest` and the corresponding value in `as`.  The--- `ne` value must be a neutral element for `f`.  If `is` has--- duplicates, `f` may be applied multiple times, and hence must be--- associative and commutative.  Out-of-bounds indices in `is` are--- ignored.+-- | `h = hist op ne k is as` computes a generalised `k`-bin histogram+-- `h`, such that `h[i]` is the sum of those values `as[j]` for which+-- `is[j]==i`.  The summation is done with `op`, which must be a+-- commutative and associative function with neutral element `ne`.  If+-- a bin has no elements, its value will be `ne`. --+-- **Work:** *O(k + n ✕ W(op))*+--+-- **Span:** *O(n ✕ W(op))* in the worst case (all updates to same+-- position), but *O(W(op))* in the best case.+--+-- In practice, the *O(n)* behaviour only occurs if *m* is also very+-- large.+def hist 'a [n] (op: a -> a -> a) (ne: a) (k: i64) (is: [n]i64) (as: [n]a) : *[k]a =+  intrinsics.hist_1d (1, map (\_ -> ne) (0..1..<k), op, ne, is, as)++-- | Like `hist`, but with initial contents of the histogram, and the+-- complexity is proportional only to the number of input elements,+-- not the total size of the histogram.+-- -- **Work:** *O(n ✕ W(op))* -- -- **Span:** *O(n ✕ W(op))* in the worst case (all updates to same@@ -132,15 +144,15 @@ -- -- In practice, the *O(n)* behaviour only occurs if *m* is also very -- large.-def reduce_by_index 'a [n] [m] (dest : *[m]a) (f : a -> a -> a) (ne : a) (is : [n]i64) (as : [n]a) : *[m]a =+def reduce_by_index 'a [k] [n] (dest : *[k]a) (f : a -> a -> a) (ne : a) (is : [n]i64) (as : [n]a) : *[k]a =   intrinsics.hist_1d (1, dest, f, ne, is, as)  -- | As `reduce_by_index`, but with two-dimensional indexes.-def reduce_by_index_2d 'a [n] [m] [k] (dest : *[m][k]a) (f : a -> a -> a) (ne : a) (is : [n](i64,i64)) (as : [n]a) : *[m][k]a =+def reduce_by_index_2d 'a [k] [n] [m] (dest : *[k][m]a) (f : a -> a -> a) (ne : a) (is : [n](i64,i64)) (as : [n]a) : *[k][m]a =   intrinsics.hist_2d (1, dest, f, ne, is, as)  -- | As `reduce_by_index`, but with three-dimensional indexes.-def reduce_by_index_3d 'a [n] [m] [k] [l] (dest : *[m][k][l]a) (f : a -> a -> a) (ne : a) (is : [n](i64,i64,i64)) (as : [n]a) : *[m][k][l]a =+def reduce_by_index_3d 'a [k] [n] [m] [l] (dest : *[k][m][l]a) (f : a -> a -> a) (ne : a) (is : [n](i64,i64,i64)) (as : [n]a) : *[k][m][l]a =   intrinsics.hist_3d (1, dest, f, ne, is, as)  -- | Inclusive prefix scan.  Has the same caveats with respect to@@ -249,35 +261,27 @@ def any [n] 'a (f: a -> bool) (as: [n]a): bool =   reduce (||) false (map f as) --- | `scatter as is vs` calculates the equivalent of this imperative--- code:------ ```--- for j in 0...length is-1:---   i = is[j]---   v = vs[j]---   if i >= 0 && i < length as:---     as[i] = v--- ```+-- | `r = spread x k is vs` produces an array `r` such that `r[i] =+-- vs[j]` where `is[j] == i`, or `x` if no such `j` exists.+-- Intuitively, `is` is an array indicating where the corresponding+-- elements of `vs` should be located in the result.  Out-of-bounds+-- elements of `is` are ignored.  In-bounds duplicates in `is` result+-- in unspecified behaviour - see `hist`@term for a function that can+-- handle this. ----- The `is` and `vs` arrays must have the same outer size.  `scatter`--- acts in-place and consumes the `as` array, returning a new array--- that has the same type and elements as `as`, except for the indices--- in `is`. Notice that writing outside the index domain of the target--- array has no effect. If `is` contains duplicates for valid indexes--- into the target array (i.e., several writes are performed to the--- same location), the result is unspecified.  It is not guaranteed--- that one of the duplicate writes will complete atomically - they--- may be interleaved.  See `reduce_by_index`@term for a function that--- can handle this case deterministically.+-- **Work:** *O(k + n)* ----- This is technically not a second-order operation, but it is defined--- here because it is closely related to the SOACs.+-- **Span:** *O(1)*+def spread 't [n] (k: i64) (x: t) (is: [n]i64) (vs: [n]t): *[k]t =+  intrinsics.scatter (map (\_ -> x) (0..1..<k), is, vs)++-- | Like `spread`, but takes an array indicating the initial values,+-- and has different work complexity. -- -- **Work:** *O(n)* -- -- **Span:** *O(1)*-def scatter 't [m] [n] (dest: *[m]t) (is: [n]i64) (vs: [n]t): *[m]t =+def scatter 't [k] [n] (dest: *[k]t) (is: [n]i64) (vs: [n]t): *[k]t =   intrinsics.scatter (dest, is, vs)  -- | `scatter_2d as is vs` is the equivalent of a `scatter` on a 2-dimensional@@ -286,7 +290,7 @@ -- **Work:** *O(n)* -- -- **Span:** *O(1)*-def scatter_2d 't [m] [n] [l] (dest: *[m][n]t) (is: [l](i64, i64)) (vs: [l]t): *[m][n]t =+def scatter_2d 't [k] [n] [l] (dest: *[k][n]t) (is: [l](i64, i64)) (vs: [l]t): *[k][n]t =   intrinsics.scatter_2d (dest, is, vs)  -- | `scatter_3d as is vs` is the equivalent of a `scatter` on a 3-dimensional@@ -295,5 +299,5 @@ -- **Work:** *O(n)* -- -- **Span:** *O(1)*-def scatter_3d 't [m] [n] [o] [l] (dest: *[m][n][o]t) (is: [l](i64, i64, i64)) (vs: [l]t): *[m][n][o]t =+def scatter_3d 't [k] [n] [o] [l] (dest: *[k][n][o]t) (is: [l](i64, i64, i64)) (vs: [l]t): *[k][n][o]t =   intrinsics.scatter_3d (dest, is, vs)

@@ -0,0 +1,457 @@+// Start of ispc_util.h.++// This header file implements various operations that are useful only when+// generating ISPC code. This includes wrappers for parts of Futhark's C runtime.++// Expose gang size+export uniform int64_t get_gang_size() {+  return programCount;+}++// Generate missing overloads for extract on pointers+#define make_extract(ty)                                                                \+static inline uniform ty * uniform extract(uniform ty * varying ptr, uniform int idx) { \+    int64 c = (int64)ptr;                                                               \+    uniform int64 r = extract(c, idx);                                                  \+    return (uniform ty * uniform)r;                                                     \+}++make_extract(int8)+make_extract(int16)+make_extract(int32)+make_extract(int64)+make_extract(uint8)+make_extract(uint16)+make_extract(uint32)+make_extract(uint64)+make_extract(float16)+make_extract(float)+make_extract(double)+make_extract(int8* uniform)+make_extract(int16* uniform)+make_extract(int32* uniform)+make_extract(int64* uniform)+make_extract(uint8* uniform)+make_extract(uint16* uniform)+make_extract(uint32* uniform)+make_extract(uint64* uniform)+make_extract(float16* uniform)+make_extract(float* uniform)+make_extract(double* uniform)+make_extract(struct futhark_context)+make_extract(struct memblock)+++// Handling of atomics+// Atomic CAS acts differently in GCC and ISPC, so we emulate it.+#define make_atomic_compare_exchange_wrapper(ty)                                     \+static inline uniform bool atomic_compare_exchange_wrapper(uniform ty * uniform mem, \+                                                           uniform ty * uniform old, \+                                                           const uniform ty val){    \+  uniform ty actual = atomic_compare_exchange_global(mem, *old, val);                \+  if (actual == *old){                                                               \+    return 1;                                                                        \+  }                                                                                  \+  *old = actual;                                                                     \+  return 0;                                                                          \+}                                                                                    \+static inline varying bool atomic_compare_exchange_wrapper(uniform ty * varying mem, \+                                                           varying ty * uniform old, \+                                                           const varying ty val){    \+  varying ty actual = atomic_compare_exchange_global(mem, *old, val);                \+  bool res = 0;                                                                      \+  if(actual == *old){                                                                \+    res = 1;                                                                         \+  } else {                                                                           \+    *old = actual;                                                                   \+  }                                                                                  \+  return res;                                                                        \+}                                                                                    \+static inline varying bool atomic_compare_exchange_wrapper(varying ty * uniform mem, \+                                                           varying ty * uniform old, \+                                                           const varying ty val){    \+  uniform ty * uniform base_mem = (uniform ty * uniform)mem;                         \+  uniform ty * uniform base_old = (uniform ty * uniform)old;                         \+  bool res = 0;                                                                      \+  foreach_active (i) {                                                               \+    uniform ty * uniform curr_mem = base_mem + i;                                    \+    uniform ty * uniform curr_old = base_old + i;                                    \+    uniform ty curr_val = extract(val, i);                                           \+    uniform bool curr = atomic_compare_exchange_wrapper(                             \+                            curr_mem, curr_old, curr_val);                           \+    res = insert(res, i, curr);                                                      \+  }                                                                                  \+  return res;                                                                        \+}                                                                                    \+static inline uniform bool atomic_compare_exchange_wrapper(uniform ty * uniform mem, \+                                                           uniform ty * uniform old, \+                                                           const varying ty val){    \+  uniform ty v = 0;                                                                  \+  foreach_active (i) v = extract(val, i);                                            \+  return atomic_compare_exchange_wrapper(mem, old, v);                               \+}++make_atomic_compare_exchange_wrapper(int32)+make_atomic_compare_exchange_wrapper(int64)+make_atomic_compare_exchange_wrapper(uint32)+make_atomic_compare_exchange_wrapper(uint64)+make_atomic_compare_exchange_wrapper(float)+make_atomic_compare_exchange_wrapper(double)++// This code generates missing overloads for atomic operations on uniform+// pointers to varying values.+#define make_single_atomic(name, ty)                                        \+static inline ty atomic_##name##_global(varying ty * uniform mem, ty val) { \+  uniform ty * uniform base_mem = (uniform ty * uniform)mem;                \+  ty res = 0;                                                               \+  foreach_active (i) {                                                      \+    uniform ty * uniform curr_mem = base_mem + i;                           \+    uniform ty curr_val = extract(val, i);                                  \+    uniform ty curr = atomic_##name##_global(curr_mem, curr_val);           \+    res = insert(res, i, curr);                                             \+  }                                                                         \+  return res;                                                               \+}++#define make_all_atomic(name)    \+make_single_atomic(name, int32)  \+make_single_atomic(name, int64)  \+make_single_atomic(name, uint32) \+make_single_atomic(name, uint64)++make_all_atomic(add)+make_all_atomic(subtract)+make_all_atomic(and)+make_all_atomic(or)+make_all_atomic(xor)+make_all_atomic(swap)++// This is a hack to prevent literals (which have unbound variability)+// from causing us to pick the wrong overload for atomic operations.+static inline varying int32  make_varying(uniform int32  x) { return x; }+static inline varying int32  make_varying(varying int32  x) { return x; }+static inline varying int64  make_varying(uniform int64  x) { return x; }+static inline varying int64  make_varying(varying int64  x) { return x; }+static inline varying uint32 make_varying(uniform uint32 x) { return x; }+static inline varying uint32 make_varying(varying uint32 x) { return x; }+static inline varying uint64 make_varying(uniform uint64 x) { return x; }+static inline varying uint64 make_varying(varying uint64 x) { return x; }++// Redirect atomic operations to the relevant ISPC overloads.+#define __atomic_fetch_add(x,y,z) atomic_add_global(x,make_varying(y))+#define __atomic_fetch_sub(x,y,z) atomic_sub_global(x,make_varying(y))+#define __atomic_fetch_and(x,y,z) atomic_and_global(x,make_varying(y))+#define __atomic_fetch_or(x,y,z) atomic_or_global(x,make_varying(y))+#define __atomic_fetch_xor(x,y,z) atomic_xor_global(x,make_varying(y))+#define __atomic_exchange_n(x,y,z) atomic_swap_global(x,make_varying(y))+#define __atomic_compare_exchange_n(x,y,z,h,j,k) atomic_compare_exchange_wrapper(x,y,z)+++// Memory allocation handling+struct memblock {+    int32_t * references;+    uint8_t * mem;+    int64_t size;+    const int8_t * desc;+};++static inline void free(void* ptr) {+  delete ptr;+}++static inline void free(void* uniform ptr) {+  delete ptr;+}++extern "C" unmasked uniform unsigned char * uniform realloc(uniform unsigned char * uniform ptr, uniform int64_t new_size);+extern "C" unmasked uniform char * uniform lexical_realloc_error(uniform int64_t new_size);+extern "C" unmasked uniform char * uniform * uniform futhark_get_error_ref(uniform struct futhark_context * uniform ctx);++static inline uniform int lexical_realloc(uniform char * uniform * uniform error,+                                          unsigned char uniform * uniform * uniform ptr,+                                          int64_t uniform * uniform old_size,+                                          uniform int64_t new_size) {+  uniform unsigned char * uniform memptr = realloc(*ptr, new_size);+  if (memptr == NULL) {+    *error = lexical_realloc_error(new_size);+    return FUTHARK_OUT_OF_MEMORY;+  } else {+    *ptr = memptr;+    *old_size = new_size;+    return FUTHARK_SUCCESS;+  }+}+++static inline uniform int lexical_realloc(uniform char * uniform * uniform error,+                                          unsigned char uniform * uniform * uniform ptr,+                                          int64_t uniform * uniform old_size,+                                          varying int64_t new_size) {+  return lexical_realloc(error, ptr, old_size, reduce_max(new_size));+}++static inline uniform int lexical_realloc(uniform char * uniform * uniform error,+                                          unsigned char uniform * varying * uniform ptr,+                                          int64_t uniform * varying old_size,+                                          varying int64_t new_size) {+  uniform int err = FUTHARK_SUCCESS;+  foreach_active(i){+    uniform unsigned char * uniform memptr = realloc(extract(*ptr,i), extract(new_size,i));+    if (memptr == NULL) {+      *error = lexical_realloc_error(extract(new_size,i));+      err = FUTHARK_OUT_OF_MEMORY;+    } else {+      *ptr = (uniform unsigned char * varying)insert((int64_t)*ptr, i, (uniform int64_t) memptr);+      *old_size = new_size;+    }+  }+  return err;+}++static inline uniform int lexical_realloc(uniform char * uniform * uniform error,+                                          unsigned char uniform * varying * uniform ptr,+                                          int64_t varying * uniform old_size,+                                          varying int64_t new_size) {+  uniform int err = FUTHARK_SUCCESS;+  foreach_active(i){+    uniform unsigned char * uniform memptr = realloc(extract(*ptr,i), extract(new_size,i));+    if (memptr == NULL) {+      *error = lexical_realloc_error(extract(new_size,i));+      err = FUTHARK_OUT_OF_MEMORY;+    } else {+      *ptr = (uniform unsigned char * varying)insert((int64_t)*ptr, i, (uniform int64_t) memptr);+      *old_size = new_size;+    }+  }+  return err;+}++static inline uniform int lexical_realloc(uniform char * uniform * uniform error,+                                          unsigned char uniform * varying * uniform ptr,+                                          size_t varying * uniform old_size,+                                          varying int64_t new_size) {+  return lexical_realloc(error, ptr, (varying int64_t * uniform)old_size, new_size);+}++static inline uniform int lexical_realloc(uniform char * uniform * uniform error,+                                          unsigned char varying * uniform * uniform ptr,+                                          size_t varying * uniform old_size,+                                          uniform int64_t new_size) {+  uniform int err = FUTHARK_SUCCESS;+  uniform unsigned char * uniform memptr = realloc((uniform unsigned char * uniform )*ptr,+                                                        new_size*programCount);+  if (memptr == NULL) {+    *error = lexical_realloc_error(new_size);+    err = FUTHARK_OUT_OF_MEMORY;+  } else {+    *ptr = (varying unsigned char * uniform)memptr;+    *old_size = new_size;+  }++  return err;+}++static inline uniform int lexical_realloc(uniform char * uniform * uniform error,+                                          unsigned char varying * uniform * uniform ptr,+                                          size_t varying * uniform old_size,+                                          varying int64_t new_size) {+  return lexical_realloc(error, ptr, old_size, reduce_max(new_size));+}++extern "C" unmasked uniform int memblock_unref(uniform struct futhark_context * uniform ctx,+                                               uniform struct memblock * uniform lhs,+                                               uniform const char * uniform lhs_desc);++static uniform int memblock_unref(uniform struct futhark_context * varying ctx,+                                  uniform struct memblock * varying lhs,+                                  uniform const char * uniform lhs_desc)+{+  uniform int err = 0;++  foreach_active(i) {+    err |= memblock_unref(extract(ctx,i), extract(lhs,i), lhs_desc);+  }++  return err;+}+static uniform int memblock_unref(uniform struct futhark_context * uniform ctx,+                                  varying struct memblock * uniform lhs,+                                  uniform const char * uniform lhs_desc)+{+  uniform int err = 0;++  varying struct memblock _lhs = *lhs;+  uniform struct memblock aos[programCount];+  aos[programIndex] = _lhs;++  foreach_active(i){+    err |= memblock_unref(ctx,+           &aos[i],+           lhs_desc);+  }++  *lhs = aos[programIndex];++  return err;+}++extern "C" unmasked uniform int memblock_alloc(uniform struct futhark_context * uniform ctx,+                                               uniform struct memblock * uniform block,+                                               uniform int64_t size,+                                               uniform const char * uniform block_desc);++static uniform int memblock_alloc(uniform struct futhark_context * varying ctx,+                                  uniform struct memblock * varying block,+                                  varying int64_t size,+                                  uniform const char * uniform block_desc) {+  uniform int err = 0;++  foreach_active(i){+    err |= memblock_alloc(extract(ctx,i), extract(block,i), extract(size, i), block_desc);+  }++  return err;+}++static uniform int memblock_alloc(uniform struct futhark_context * uniform ctx,+                                  varying struct memblock * uniform block,+                                  uniform int64_t size,+                                  uniform const char * uniform block_desc) {+  uniform int err = 0;++  varying struct memblock _block = *block;+  uniform struct memblock aos[programCount];+  aos[programIndex] = _block;++  foreach_active(i){+    err |= memblock_alloc(ctx, &aos[i], size, block_desc);+  }+  *block = aos[programIndex];++  return err;+}++static uniform int memblock_alloc(uniform struct futhark_context * uniform ctx,+                                  varying struct memblock * uniform block,+                                  varying int64_t size,+                                  uniform const char * uniform block_desc) {+  uniform int err = 0;++  varying struct memblock _block = *block;+  uniform struct memblock aos[programCount];+  aos[programIndex] = _block;+  foreach_active(i){+    err |= memblock_alloc(ctx, &aos[i], extract(size, i), block_desc);+  }+  *block = aos[programIndex];++  return err;+}++extern "C" unmasked uniform int memblock_set(uniform struct futhark_context * uniform ctx,+                                             uniform struct memblock * uniform lhs,+                                             uniform struct memblock * uniform rhs,+                                             uniform const char * uniform lhs_desc);++static uniform int memblock_set (uniform struct futhark_context * uniform ctx,+                                 varying struct memblock * uniform lhs,+                                 varying struct memblock * uniform rhs,+                                 uniform const char * uniform lhs_desc) {+  uniform int err = 0;++  varying struct memblock _lhs = *lhs;+  varying struct memblock _rhs = *rhs;+  uniform struct memblock aos1[programCount];+  aos1[programIndex] = _lhs;++  uniform struct memblock aos2[programCount];+  aos2[programIndex] = _rhs;++  foreach_active(i) {+      err |= memblock_set(ctx,+      &aos1[i],+      &aos2[i],+      lhs_desc);+  }+  *lhs = aos1[programIndex];+  *rhs = aos2[programIndex];++  return err;+}++static uniform int memblock_set (uniform struct futhark_context * uniform ctx,+                                 varying struct memblock * uniform lhs,+                                 uniform struct memblock * uniform rhs,+                                 uniform const char * uniform lhs_desc) {+  uniform int err = 0;++  varying struct memblock _lhs = *lhs;+  uniform struct memblock aos1[programCount];+  aos1[programIndex] = _lhs;++  foreach_active(i) {+      err |= memblock_set(ctx,+      &aos1[i],+      rhs,+      lhs_desc);+  }+  *lhs = aos1[programIndex];++  return err;+}++// AOS <-> SOA memcpy functions+#define memmove_sized(dim)                                                                                      \+static inline void memmove_##dim(varying uint8 * uniform dst, uniform uint8 * varying src, uniform int64_t n) { \+    uniform uint##dim * varying srcp = (uniform uint##dim * varying) src;                                       \+    varying uint##dim * uniform dstp = (varying uint##dim * uniform) dst;                                       \+    for (uniform int64_t i = 0; i < n / (dim / 8); i++) {                                                       \+        dstp[i] = srcp[i];                                                                                      \+    }                                                                                                           \+}                                                                                                               \+static inline void memmove_##dim(uniform uint8 * varying dst, varying uint8 * uniform src, uniform int64_t n) { \+    varying uint##dim * uniform srcp = (varying uint##dim * uniform) src;                                       \+    uniform uint##dim * varying dstp = (uniform uint##dim * varying) dst;                                       \+    for (uniform int64_t i = 0; i < n / (dim / 8); i++) {                                                       \+        dstp[i] = srcp[i];                                                                                      \+    }                                                                                                           \+}                                                                                                               \+static inline void memmove_##dim(varying uint8 * uniform dst, varying uint8 * uniform src, uniform int64_t n) { \+    varying uint##dim * uniform srcp = (varying uint##dim * uniform) src;                                       \+    varying uint##dim * uniform dstp = (varying uint##dim * uniform) dst;                                       \+    for (uniform int64_t i = 0; i < n / (dim / 8); i++) {                                                       \+        dstp[i] = srcp[i];                                                                                      \+    }                                                                                                           \+}                                                                                                               \+static inline void memmove_##dim(varying uint8 * varying dst, uniform uint8 * varying src, uniform int64_t n) { \+    foreach_unique (ptr in dst) {                                                                               \+        memmove_##dim(ptr, src, n);                                                                             \+    }                                                                                                           \+}                                                                                                               \+static inline void memmove_##dim(uniform uint8 * varying dst, varying uint8 * varying src, uniform int64_t n) { \+    foreach_unique (ptr in src) {                                                                               \+        memmove_##dim(dst, ptr, n);                                                                             \+    }                                                                                                           \+}                                                                                                               \+static inline void memmove_##dim(varying uint8 * varying dst, varying uint8 * uniform src, uniform int64_t n) { \+    foreach_unique (ptr in dst) {                                                                               \+        memmove_##dim(ptr, src, n);                                                                             \+    }                                                                                                           \+}                                                                                                               \+static inline void memmove_##dim(varying uint8 * varying dst, varying uint8 * varying src, uniform int64_t n) { \+    if (reduce_equal((varying int64_t)dst)) {                                                                   \+        foreach_unique (ptr in src) {                                                                           \+            memmove_##dim(dst, ptr, n);                                                                         \+        }                                                                                                       \+    } else {                                                                                                    \+        foreach_unique (ptr in dst) {                                                                           \+            memmove_##dim(ptr, src, n);                                                                         \+        }                                                                                                       \+    }                                                                                                           \+}+memmove_sized(8)+memmove_sized(16)+memmove_sized(32)+memmove_sized(64)++// End of ispc_util.h.

@@ -65,1705 +65,2929 @@   return x * y; } -static inline uint8_t udiv8(uint8_t x, uint8_t y) {-  return x / y;-}--static inline uint16_t udiv16(uint16_t x, uint16_t y) {-  return x / y;-}--static inline uint32_t udiv32(uint32_t x, uint32_t y) {-  return x / y;-}--static inline uint64_t udiv64(uint64_t x, uint64_t y) {-  return x / y;-}--static inline uint8_t udiv_up8(uint8_t x, uint8_t y) {-  return (x + y - 1) / y;-}--static inline uint16_t udiv_up16(uint16_t x, uint16_t y) {-  return (x + y - 1) / y;-}--static inline uint32_t udiv_up32(uint32_t x, uint32_t y) {-  return (x + y - 1) / y;-}--static inline uint64_t udiv_up64(uint64_t x, uint64_t y) {-  return (x + y - 1) / y;-}--static inline uint8_t umod8(uint8_t x, uint8_t y) {-  return x % y;-}--static inline uint16_t umod16(uint16_t x, uint16_t y) {-  return x % y;-}--static inline uint32_t umod32(uint32_t x, uint32_t y) {-  return x % y;-}--static inline uint64_t umod64(uint64_t x, uint64_t y) {-  return x % y;-}--static inline uint8_t udiv_safe8(uint8_t x, uint8_t y) {-  return y == 0 ? 0 : x / y;-}--static inline uint16_t udiv_safe16(uint16_t x, uint16_t y) {-  return y == 0 ? 0 : x / y;-}--static inline uint32_t udiv_safe32(uint32_t x, uint32_t y) {-  return y == 0 ? 0 : x / y;-}--static inline uint64_t udiv_safe64(uint64_t x, uint64_t y) {-  return y == 0 ? 0 : x / y;-}--static inline uint8_t udiv_up_safe8(uint8_t x, uint8_t y) {-  return y == 0 ? 0 : (x + y - 1) / y;-}--static inline uint16_t udiv_up_safe16(uint16_t x, uint16_t y) {-  return y == 0 ? 0 : (x + y - 1) / y;-}--static inline uint32_t udiv_up_safe32(uint32_t x, uint32_t y) {-  return y == 0 ? 0 : (x + y - 1) / y;-}--static inline uint64_t udiv_up_safe64(uint64_t x, uint64_t y) {-  return y == 0 ? 0 : (x + y - 1) / y;-}--static inline uint8_t umod_safe8(uint8_t x, uint8_t y) {-  return y == 0 ? 0 : x % y;-}--static inline uint16_t umod_safe16(uint16_t x, uint16_t y) {-  return y == 0 ? 0 : x % y;-}--static inline uint32_t umod_safe32(uint32_t x, uint32_t y) {-  return y == 0 ? 0 : x % y;-}--static inline uint64_t umod_safe64(uint64_t x, uint64_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int8_t sdiv8(int8_t x, int8_t y) {-  int8_t q = x / y;-  int8_t r = x % y;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int16_t sdiv16(int16_t x, int16_t y) {-  int16_t q = x / y;-  int16_t r = x % y;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int32_t sdiv32(int32_t x, int32_t y) {-  int32_t q = x / y;-  int32_t r = x % y;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int64_t sdiv64(int64_t x, int64_t y) {-  int64_t q = x / y;-  int64_t r = x % y;--  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);-}--static inline int8_t sdiv_up8(int8_t x, int8_t y) {-  return sdiv8(x + y - 1, y);-}--static inline int16_t sdiv_up16(int16_t x, int16_t y) {-  return sdiv16(x + y - 1, y);-}--static inline int32_t sdiv_up32(int32_t x, int32_t y) {-  return sdiv32(x + y - 1, y);-}--static inline int64_t sdiv_up64(int64_t x, int64_t y) {-  return sdiv64(x + y - 1, y);-}--static inline int8_t smod8(int8_t x, int8_t y) {-  int8_t r = x % y;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int16_t smod16(int16_t x, int16_t y) {-  int16_t r = x % y;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int32_t smod32(int32_t x, int32_t y) {-  int32_t r = x % y;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int64_t smod64(int64_t x, int64_t y) {-  int64_t r = x % y;--  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);-}--static inline int8_t sdiv_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : sdiv8(x, y);-}--static inline int16_t sdiv_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : sdiv16(x, y);-}--static inline int32_t sdiv_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : sdiv32(x, y);-}--static inline int64_t sdiv_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : sdiv64(x, y);-}--static inline int8_t sdiv_up_safe8(int8_t x, int8_t y) {-  return sdiv_safe8(x + y - 1, y);-}--static inline int16_t sdiv_up_safe16(int16_t x, int16_t y) {-  return sdiv_safe16(x + y - 1, y);-}--static inline int32_t sdiv_up_safe32(int32_t x, int32_t y) {-  return sdiv_safe32(x + y - 1, y);-}--static inline int64_t sdiv_up_safe64(int64_t x, int64_t y) {-  return sdiv_safe64(x + y - 1, y);-}--static inline int8_t smod_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : smod8(x, y);-}--static inline int16_t smod_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : smod16(x, y);-}--static inline int32_t smod_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : smod32(x, y);-}--static inline int64_t smod_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : smod64(x, y);-}--static inline int8_t squot8(int8_t x, int8_t y) {-  return x / y;-}--static inline int16_t squot16(int16_t x, int16_t y) {-  return x / y;-}--static inline int32_t squot32(int32_t x, int32_t y) {-  return x / y;-}--static inline int64_t squot64(int64_t x, int64_t y) {-  return x / y;-}--static inline int8_t srem8(int8_t x, int8_t y) {-  return x % y;-}--static inline int16_t srem16(int16_t x, int16_t y) {-  return x % y;-}--static inline int32_t srem32(int32_t x, int32_t y) {-  return x % y;-}--static inline int64_t srem64(int64_t x, int64_t y) {-  return x % y;-}--static inline int8_t squot_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : x / y;-}--static inline int16_t squot_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : x / y;-}--static inline int32_t squot_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : x / y;-}--static inline int64_t squot_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : x / y;-}--static inline int8_t srem_safe8(int8_t x, int8_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int16_t srem_safe16(int16_t x, int16_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int32_t srem_safe32(int32_t x, int32_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int64_t srem_safe64(int64_t x, int64_t y) {-  return y == 0 ? 0 : x % y;-}--static inline int8_t smin8(int8_t x, int8_t y) {-  return x < y ? x : y;-}--static inline int16_t smin16(int16_t x, int16_t y) {-  return x < y ? x : y;-}--static inline int32_t smin32(int32_t x, int32_t y) {-  return x < y ? x : y;-}--static inline int64_t smin64(int64_t x, int64_t y) {-  return x < y ? x : y;-}--static inline uint8_t umin8(uint8_t x, uint8_t y) {-  return x < y ? x : y;-}--static inline uint16_t umin16(uint16_t x, uint16_t y) {-  return x < y ? x : y;-}--static inline uint32_t umin32(uint32_t x, uint32_t y) {-  return x < y ? x : y;-}--static inline uint64_t umin64(uint64_t x, uint64_t y) {-  return x < y ? x : y;-}--static inline int8_t smax8(int8_t x, int8_t y) {-  return x < y ? y : x;-}--static inline int16_t smax16(int16_t x, int16_t y) {-  return x < y ? y : x;-}--static inline int32_t smax32(int32_t x, int32_t y) {-  return x < y ? y : x;-}--static inline int64_t smax64(int64_t x, int64_t y) {-  return x < y ? y : x;-}--static inline uint8_t umax8(uint8_t x, uint8_t y) {-  return x < y ? y : x;-}--static inline uint16_t umax16(uint16_t x, uint16_t y) {-  return x < y ? y : x;-}--static inline uint32_t umax32(uint32_t x, uint32_t y) {-  return x < y ? y : x;-}--static inline uint64_t umax64(uint64_t x, uint64_t y) {-  return x < y ? y : x;-}--static inline uint8_t shl8(uint8_t x, uint8_t y) {-  return (uint8_t)(x << y);-}--static inline uint16_t shl16(uint16_t x, uint16_t y) {-  return (uint16_t)(x << y);-}--static inline uint32_t shl32(uint32_t x, uint32_t y) {-  return x << y;-}--static inline uint64_t shl64(uint64_t x, uint64_t y) {-  return x << y;-}--static inline uint8_t lshr8(uint8_t x, uint8_t y) {-  return x >> y;-}--static inline uint16_t lshr16(uint16_t x, uint16_t y) {-  return x >> y;-}--static inline uint32_t lshr32(uint32_t x, uint32_t y) {-  return x >> y;-}--static inline uint64_t lshr64(uint64_t x, uint64_t y) {-  return x >> y;-}--static inline int8_t ashr8(int8_t x, int8_t y) {-  return x >> y;-}--static inline int16_t ashr16(int16_t x, int16_t y) {-  return x >> y;-}--static inline int32_t ashr32(int32_t x, int32_t y) {-  return x >> y;-}--static inline int64_t ashr64(int64_t x, int64_t y) {-  return x >> y;-}--static inline uint8_t and8(uint8_t x, uint8_t y) {-  return x & y;-}--static inline uint16_t and16(uint16_t x, uint16_t y) {-  return x & y;-}--static inline uint32_t and32(uint32_t x, uint32_t y) {-  return x & y;-}--static inline uint64_t and64(uint64_t x, uint64_t y) {-  return x & y;-}--static inline uint8_t or8(uint8_t x, uint8_t y) {-  return x | y;-}--static inline uint16_t or16(uint16_t x, uint16_t y) {-  return x | y;-}--static inline uint32_t or32(uint32_t x, uint32_t y) {-  return x | y;-}--static inline uint64_t or64(uint64_t x, uint64_t y) {-  return x | y;-}--static inline uint8_t xor8(uint8_t x, uint8_t y) {-  return x ^ y;-}--static inline uint16_t xor16(uint16_t x, uint16_t y) {-  return x ^ y;-}--static inline uint32_t xor32(uint32_t x, uint32_t y) {-  return x ^ y;-}--static inline uint64_t xor64(uint64_t x, uint64_t y) {-  return x ^ y;-}--static inline bool ult8(uint8_t x, uint8_t y) {-  return x < y;-}--static inline bool ult16(uint16_t x, uint16_t y) {-  return x < y;-}--static inline bool ult32(uint32_t x, uint32_t y) {-  return x < y;-}--static inline bool ult64(uint64_t x, uint64_t y) {-  return x < y;-}--static inline bool ule8(uint8_t x, uint8_t y) {-  return x <= y;-}--static inline bool ule16(uint16_t x, uint16_t y) {-  return x <= y;-}--static inline bool ule32(uint32_t x, uint32_t y) {-  return x <= y;-}--static inline bool ule64(uint64_t x, uint64_t y) {-  return x <= y;-}--static inline bool slt8(int8_t x, int8_t y) {-  return x < y;-}--static inline bool slt16(int16_t x, int16_t y) {-  return x < y;-}--static inline bool slt32(int32_t x, int32_t y) {-  return x < y;-}--static inline bool slt64(int64_t x, int64_t y) {-  return x < y;-}--static inline bool sle8(int8_t x, int8_t y) {-  return x <= y;-}--static inline bool sle16(int16_t x, int16_t y) {-  return x <= y;-}--static inline bool sle32(int32_t x, int32_t y) {-  return x <= y;-}--static inline bool sle64(int64_t x, int64_t y) {-  return x <= y;-}--static inline uint8_t pow8(uint8_t x, uint8_t y) {-  uint8_t res = 1, rem = y;--  while (rem != 0) {-    if (rem & 1)-      res *= x;-    rem >>= 1;-    x *= x;-  }-  return res;-}--static inline uint16_t pow16(uint16_t x, uint16_t y) {-  uint16_t res = 1, rem = y;--  while (rem != 0) {-    if (rem & 1)-      res *= x;-    rem >>= 1;-    x *= x;-  }-  return res;-}--static inline uint32_t pow32(uint32_t x, uint32_t y) {-  uint32_t res = 1, rem = y;--  while (rem != 0) {-    if (rem & 1)-      res *= x;-    rem >>= 1;-    x *= x;-  }-  return res;-}--static inline uint64_t pow64(uint64_t x, uint64_t y) {-  uint64_t res = 1, rem = y;--  while (rem != 0) {-    if (rem & 1)-      res *= x;-    rem >>= 1;-    x *= x;-  }-  return res;-}--static inline bool itob_i8_bool(int8_t x) {-  return x;-}--static inline bool itob_i16_bool(int16_t x) {-  return x;-}--static inline bool itob_i32_bool(int32_t x) {-  return x;-}--static inline bool itob_i64_bool(int64_t x) {-  return x;-}--static inline int8_t btoi_bool_i8(bool x) {-  return x;-}--static inline int16_t btoi_bool_i16(bool x) {-  return x;-}--static inline int32_t btoi_bool_i32(bool x) {-  return x;-}--static inline int64_t btoi_bool_i64(bool x) {-  return x;-}--#define sext_i8_i8(x) ((int8_t) (int8_t) (x))-#define sext_i8_i16(x) ((int16_t) (int8_t) (x))-#define sext_i8_i32(x) ((int32_t) (int8_t) (x))-#define sext_i8_i64(x) ((int64_t) (int8_t) (x))-#define sext_i16_i8(x) ((int8_t) (int16_t) (x))-#define sext_i16_i16(x) ((int16_t) (int16_t) (x))-#define sext_i16_i32(x) ((int32_t) (int16_t) (x))-#define sext_i16_i64(x) ((int64_t) (int16_t) (x))-#define sext_i32_i8(x) ((int8_t) (int32_t) (x))-#define sext_i32_i16(x) ((int16_t) (int32_t) (x))-#define sext_i32_i32(x) ((int32_t) (int32_t) (x))-#define sext_i32_i64(x) ((int64_t) (int32_t) (x))-#define sext_i64_i8(x) ((int8_t) (int64_t) (x))-#define sext_i64_i16(x) ((int16_t) (int64_t) (x))-#define sext_i64_i32(x) ((int32_t) (int64_t) (x))-#define sext_i64_i64(x) ((int64_t) (int64_t) (x))-#define zext_i8_i8(x) ((int8_t) (uint8_t) (x))-#define zext_i8_i16(x) ((int16_t) (uint8_t) (x))-#define zext_i8_i32(x) ((int32_t) (uint8_t) (x))-#define zext_i8_i64(x) ((int64_t) (uint8_t) (x))-#define zext_i16_i8(x) ((int8_t) (uint16_t) (x))-#define zext_i16_i16(x) ((int16_t) (uint16_t) (x))-#define zext_i16_i32(x) ((int32_t) (uint16_t) (x))-#define zext_i16_i64(x) ((int64_t) (uint16_t) (x))-#define zext_i32_i8(x) ((int8_t) (uint32_t) (x))-#define zext_i32_i16(x) ((int16_t) (uint32_t) (x))-#define zext_i32_i32(x) ((int32_t) (uint32_t) (x))-#define zext_i32_i64(x) ((int64_t) (uint32_t) (x))-#define zext_i64_i8(x) ((int8_t) (uint64_t) (x))-#define zext_i64_i16(x) ((int16_t) (uint64_t) (x))-#define zext_i64_i32(x) ((int32_t) (uint64_t) (x))-#define zext_i64_i64(x) ((int64_t) (uint64_t) (x))--static int8_t abs8(int8_t x) {-  return (int8_t)abs(x);-}--static int16_t abs16(int16_t x) {-  return (int16_t)abs(x);-}--static int32_t abs32(int32_t x) {-  return abs(x);-}--static int64_t abs64(int64_t x) {-#if defined(__OPENCL_VERSION__)-  return abs(x);-#else-  return llabs(x);-#endif-}--#if defined(__OPENCL_VERSION__)-static int32_t futrts_popc8(int8_t x) {-  return popcount(x);-}--static int32_t futrts_popc16(int16_t x) {-  return popcount(x);-}--static int32_t futrts_popc32(int32_t x) {-  return popcount(x);-}--static int32_t futrts_popc64(int64_t x) {-  return popcount(x);-}-#elif defined(__CUDA_ARCH__)--static int32_t futrts_popc8(int8_t x) {-  return __popc(zext_i8_i32(x));-}--static int32_t futrts_popc16(int16_t x) {-  return __popc(zext_i16_i32(x));-}--static int32_t futrts_popc32(int32_t x) {-  return __popc(x);-}--static int32_t futrts_popc64(int64_t x) {-  return __popcll(x);-}--#else // Not OpenCL or CUDA, but plain C.--static int32_t futrts_popc8(uint8_t x) {-  int c = 0;-  for (; x; ++c) { x &= x - 1; }-  return c;-}--static int32_t futrts_popc16(uint16_t x) {-  int c = 0;-  for (; x; ++c) { x &= x - 1; }-  return c;-}--static int32_t futrts_popc32(uint32_t x) {-  int c = 0;-  for (; x; ++c) { x &= x - 1; }-  return c;-}--static int32_t futrts_popc64(uint64_t x) {-  int c = 0;-  for (; x; ++c) { x &= x - 1; }-  return c;-}-#endif--#if defined(__OPENCL_VERSION__)-static uint8_t futrts_mul_hi8(uint8_t a, uint8_t b) {-  return mul_hi(a, b);-}--static uint16_t futrts_mul_hi16(uint16_t a, uint16_t b) {-  return mul_hi(a, b);-}--static uint32_t futrts_mul_hi32(uint32_t a, uint32_t b) {-  return mul_hi(a, b);-}--static uint64_t futrts_mul_hi64(uint64_t a, uint64_t b) {-  return mul_hi(a, b);-}--#elif defined(__CUDA_ARCH__)--static uint8_t futrts_mul_hi8(uint8_t a, uint8_t b) {-  uint16_t aa = a;-  uint16_t bb = b;--  return aa * bb >> 8;-}--static uint16_t futrts_mul_hi16(uint16_t a, uint16_t b) {-  uint32_t aa = a;-  uint32_t bb = b;--  return aa * bb >> 16;-}--static uint32_t futrts_mul_hi32(uint32_t a, uint32_t b) {-  return mulhi(a, b);-}--static uint64_t futrts_mul_hi64(uint64_t a, uint64_t b) {-  return mul64hi(a, b);-}--#else // Not OpenCL or CUDA, but plain C.--static uint8_t futrts_mul_hi8(uint8_t a, uint8_t b) {-  uint16_t aa = a;-  uint16_t bb = b;--  return aa * bb >> 8;-}--static uint16_t futrts_mul_hi16(uint16_t a, uint16_t b) {-  uint32_t aa = a;-  uint32_t bb = b;--  return aa * bb >> 16;-}--static uint32_t futrts_mul_hi32(uint32_t a, uint32_t b) {-  uint64_t aa = a;-  uint64_t bb = b;--  return aa * bb >> 32;-}--static uint64_t futrts_mul_hi64(uint64_t a, uint64_t b) {-  __uint128_t aa = a;-  __uint128_t bb = b;--  return aa * bb >> 64;-}-#endif--#if defined(__OPENCL_VERSION__)-static uint8_t futrts_mad_hi8(uint8_t a, uint8_t b, uint8_t c) {-  return mad_hi(a, b, c);-}--static uint16_t futrts_mad_hi16(uint16_t a, uint16_t b, uint16_t c) {-  return mad_hi(a, b, c);-}--static uint32_t futrts_mad_hi32(uint32_t a, uint32_t b, uint32_t c) {-  return mad_hi(a, b, c);-}--static uint64_t futrts_mad_hi64(uint64_t a, uint64_t b, uint64_t c) {-  return mad_hi(a, b, c);-}--#else // Not OpenCL--static uint8_t futrts_mad_hi8(uint8_t a, uint8_t b, uint8_t c) {-  return futrts_mul_hi8(a, b) + c;-}--static uint16_t futrts_mad_hi16(uint16_t a, uint16_t b, uint16_t c) {-  return futrts_mul_hi16(a, b) + c;-}--static uint32_t futrts_mad_hi32(uint32_t a, uint32_t b, uint32_t c) {-  return futrts_mul_hi32(a, b) + c;-}--static uint64_t futrts_mad_hi64(uint64_t a, uint64_t b, uint64_t c) {-  return futrts_mul_hi64(a, b) + c;-}-#endif--#if defined(__OPENCL_VERSION__)-static int32_t futrts_clzz8(int8_t x) {-  return clz(x);-}--static int32_t futrts_clzz16(int16_t x) {-  return clz(x);-}--static int32_t futrts_clzz32(int32_t x) {-  return clz(x);-}--static int32_t futrts_clzz64(int64_t x) {-  return clz(x);-}--#elif defined(__CUDA_ARCH__)--static int32_t futrts_clzz8(int8_t x) {-  return __clz(zext_i8_i32(x)) - 24;-}--static int32_t futrts_clzz16(int16_t x) {-  return __clz(zext_i16_i32(x)) - 16;-}--static int32_t futrts_clzz32(int32_t x) {-  return __clz(x);-}--static int32_t futrts_clzz64(int64_t x) {-  return __clzll(x);-}--#else // Not OpenCL or CUDA, but plain C.--static int32_t futrts_clzz8(int8_t x) {-  return x == 0 ? 8 : __builtin_clz((uint32_t)zext_i8_i32(x)) - 24;-}--static int32_t futrts_clzz16(int16_t x) {-  return x == 0 ? 16 : __builtin_clz((uint32_t)zext_i16_i32(x)) - 16;-}--static int32_t futrts_clzz32(int32_t x) {-  return x == 0 ? 32 : __builtin_clz((uint32_t)x);-}--static int32_t futrts_clzz64(int64_t x) {-  return x == 0 ? 64 : __builtin_clzll((uint64_t)x);-}-#endif--#if defined(__OPENCL_VERSION__)-static int32_t futrts_ctzz8(int8_t x) {-  int i = 0;-  for (; i < 8 && (x & 1) == 0; i++, x >>= 1)-    ;-  return i;-}--static int32_t futrts_ctzz16(int16_t x) {-  int i = 0;-  for (; i < 16 && (x & 1) == 0; i++, x >>= 1)-    ;-  return i;-}--static int32_t futrts_ctzz32(int32_t x) {-  int i = 0;-  for (; i < 32 && (x & 1) == 0; i++, x >>= 1)-    ;-  return i;-}--static int32_t futrts_ctzz64(int64_t x) {-  int i = 0;-  for (; i < 64 && (x & 1) == 0; i++, x >>= 1)-    ;-  return i;-}--#elif defined(__CUDA_ARCH__)--static int32_t futrts_ctzz8(int8_t x) {-  int y = __ffs(x);-  return y == 0 ? 8 : y - 1;-}--static int32_t futrts_ctzz16(int16_t x) {-  int y = __ffs(x);-  return y == 0 ? 16 : y - 1;-}--static int32_t futrts_ctzz32(int32_t x) {-  int y = __ffs(x);-  return y == 0 ? 32 : y - 1;-}--static int32_t futrts_ctzz64(int64_t x) {-  int y = __ffsll(x);-  return y == 0 ? 64 : y - 1;-}--#else // Not OpenCL or CUDA, but plain C.--static int32_t futrts_ctzz8(int8_t x) {-  return x == 0 ? 8 : __builtin_ctz((uint32_t)x);-}--static int32_t futrts_ctzz16(int16_t x) {-  return x == 0 ? 16 : __builtin_ctz((uint32_t)x);-}--static int32_t futrts_ctzz32(int32_t x) {-  return x == 0 ? 32 : __builtin_ctz((uint32_t)x);-}--static int32_t futrts_ctzz64(int64_t x) {-  return x == 0 ? 64 : __builtin_ctzll((uint64_t)x);-}-#endif--static inline float fdiv32(float x, float y) {-  return x / y;-}--static inline float fadd32(float x, float y) {-  return x + y;-}--static inline float fsub32(float x, float y) {-  return x - y;-}--static inline float fmul32(float x, float y) {-  return x * y;-}--static inline bool cmplt32(float x, float y) {-  return x < y;-}--static inline bool cmple32(float x, float y) {-  return x <= y;-}--static inline float sitofp_i8_f32(int8_t x) {-  return (float) x;-}--static inline float sitofp_i16_f32(int16_t x) {-  return (float) x;-}--static inline float sitofp_i32_f32(int32_t x) {-  return (float) x;-}--static inline float sitofp_i64_f32(int64_t x) {-  return (float) x;-}--static inline float uitofp_i8_f32(uint8_t x) {-  return (float) x;-}--static inline float uitofp_i16_f32(uint16_t x) {-  return (float) x;-}--static inline float uitofp_i32_f32(uint32_t x) {-  return (float) x;-}--static inline float uitofp_i64_f32(uint64_t x) {-  return (float) x;-}--#ifdef __OPENCL_VERSION__-static inline float fabs32(float x) {-  return fabs(x);-}--static inline float fmax32(float x, float y) {-  return fmax(x, y);-}--static inline float fmin32(float x, float y) {-  return fmin(x, y);-}--static inline float fpow32(float x, float y) {-  return pow(x, y);-}--#else // Not OpenCL, but CUDA or plain C.--static inline float fabs32(float x) {-  return fabsf(x);-}--static inline float fmax32(float x, float y) {-  return fmaxf(x, y);-}--static inline float fmin32(float x, float y) {-  return fminf(x, y);-}--static inline float fpow32(float x, float y) {-  return powf(x, y);-}-#endif--static inline bool futrts_isnan32(float x) {-  return isnan(x);-}--static inline bool futrts_isinf32(float x) {-  return isinf(x);-}--static inline int8_t fptosi_f32_i8(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (int8_t) x;-  }-}--static inline int16_t fptosi_f32_i16(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (int16_t) x;-  }-}--static inline int32_t fptosi_f32_i32(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (int32_t) x;-  }-}--static inline int64_t fptosi_f32_i64(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (int64_t) x;-  };-}--static inline uint8_t fptoui_f32_i8(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (uint8_t) (int8_t) x;-  }-}--static inline uint16_t fptoui_f32_i16(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (uint16_t) (int16_t) x;-  }-}--static inline uint32_t fptoui_f32_i32(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (uint32_t) (int32_t) x;-  }-}--static inline uint64_t fptoui_f32_i64(float x) {-  if (futrts_isnan32(x) || futrts_isinf32(x)) {-    return 0;-  } else {-    return (uint64_t) (int64_t) x;-  }-}--static inline bool ftob_f32_bool(float x) {-  return x != 0;-}--static inline float btof_bool_f32(bool x) {-  return x ? 1 : 0;-}--#ifdef __OPENCL_VERSION__-static inline float futrts_log32(float x) {-  return log(x);-}--static inline float futrts_log2_32(float x) {-  return log2(x);-}--static inline float futrts_log10_32(float x) {-  return log10(x);-}--static inline float futrts_sqrt32(float x) {-  return sqrt(x);-}--static inline float futrts_cbrt32(float x) {-  return cbrt(x);-}--static inline float futrts_exp32(float x) {-  return exp(x);-}--static inline float futrts_cos32(float x) {-  return cos(x);-}--static inline float futrts_sin32(float x) {-  return sin(x);-}--static inline float futrts_tan32(float x) {-  return tan(x);-}--static inline float futrts_acos32(float x) {-  return acos(x);-}--static inline float futrts_asin32(float x) {-  return asin(x);-}--static inline float futrts_atan32(float x) {-  return atan(x);-}--static inline float futrts_cosh32(float x) {-  return cosh(x);-}--static inline float futrts_sinh32(float x) {-  return sinh(x);-}--static inline float futrts_tanh32(float x) {-  return tanh(x);-}--static inline float futrts_acosh32(float x) {-  return acosh(x);-}--static inline float futrts_asinh32(float x) {-  return asinh(x);-}--static inline float futrts_atanh32(float x) {-  return atanh(x);-}--static inline float futrts_atan2_32(float x, float y) {-  return atan2(x, y);-}--static inline float futrts_hypot32(float x, float y) {-  return hypot(x, y);-}--static inline float futrts_gamma32(float x) {-  return tgamma(x);-}--static inline float futrts_lgamma32(float x) {-  return lgamma(x);-}--static inline float futrts_erf32(float x) {-  return erf(x);-}--static inline float futrts_erfc32(float x) {-  return erfc(x);-}--static inline float fmod32(float x, float y) {-  return fmod(x, y);-}--static inline float futrts_round32(float x) {-  return rint(x);-}--static inline float futrts_floor32(float x) {-  return floor(x);-}--static inline float futrts_ceil32(float x) {-  return ceil(x);-}--static inline float futrts_lerp32(float v0, float v1, float t) {-  return mix(v0, v1, t);-}--static inline float futrts_mad32(float a, float b, float c) {-  return mad(a, b, c);-}--static inline float futrts_fma32(float a, float b, float c) {-  return fma(a, b, c);-}--#else // Not OpenCL, but CUDA or plain C.--static inline float futrts_log32(float x) {-  return logf(x);-}--static inline float futrts_log2_32(float x) {-  return log2f(x);-}--static inline float futrts_log10_32(float x) {-  return log10f(x);-}--static inline float futrts_sqrt32(float x) {-  return sqrtf(x);-}--static inline float futrts_cbrt32(float x) {-  return cbrtf(x);-}--static inline float futrts_exp32(float x) {-  return expf(x);-}--static inline float futrts_cos32(float x) {-  return cosf(x);-}--static inline float futrts_sin32(float x) {-  return sinf(x);-}--static inline float futrts_tan32(float x) {-  return tanf(x);-}--static inline float futrts_acos32(float x) {-  return acosf(x);-}--static inline float futrts_asin32(float x) {-  return asinf(x);-}--static inline float futrts_atan32(float x) {-  return atanf(x);-}--static inline float futrts_cosh32(float x) {-  return coshf(x);-}--static inline float futrts_sinh32(float x) {-  return sinhf(x);-}--static inline float futrts_tanh32(float x) {-  return tanhf(x);-}--static inline float futrts_acosh32(float x) {-  return acoshf(x);-}--static inline float futrts_asinh32(float x) {-  return asinhf(x);-}--static inline float futrts_atanh32(float x) {-  return atanhf(x);-}--static inline float futrts_atan2_32(float x, float y) {-  return atan2f(x, y);-}--static inline float futrts_hypot32(float x, float y) {-  return hypotf(x, y);-}--static inline float futrts_gamma32(float x) {-  return tgammaf(x);-}--static inline float futrts_lgamma32(float x) {-  return lgammaf(x);-}--static inline float futrts_erf32(float x) {-  return erff(x);-}--static inline float futrts_erfc32(float x) {-  return erfcf(x);-}--static inline float fmod32(float x, float y) {-  return fmodf(x, y);-}--static inline float futrts_round32(float x) {-  return rintf(x);-}--static inline float futrts_floor32(float x) {-  return floorf(x);-}--static inline float futrts_ceil32(float x) {-  return ceilf(x);-}--static inline float futrts_lerp32(float v0, float v1, float t) {-  return v0 + (v1 - v0) * t;-}--static inline float futrts_mad32(float a, float b, float c) {-  return a * b + c;-}--static inline float futrts_fma32(float a, float b, float c) {-  return fmaf(a, b, c);-}-#endif--static inline int32_t futrts_to_bits32(float x) {-  union {-    float f;-    int32_t t;-  } p;--  p.f = x;-  return p.t;-}--static inline float futrts_from_bits32(int32_t x) {-  union {-    int32_t f;-    float t;-  } p;--  p.f = x;-  return p.t;-}--static inline float fsignum32(float x) {-  return futrts_isnan32(x) ? x : (x > 0) - (x < 0);-}--#ifdef FUTHARK_F64_ENABLED--static inline double fdiv64(double x, double y) {-  return x / y;-}--static inline double fadd64(double x, double y) {-  return x + y;-}--static inline double fsub64(double x, double y) {-  return x - y;-}--static inline double fmul64(double x, double y) {-  return x * y;-}--static inline bool cmplt64(double x, double y) {-  return x < y;-}--static inline bool cmple64(double x, double y) {-  return x <= y;-}--static inline double sitofp_i8_f64(int8_t x) {-  return (double) x;-}--static inline double sitofp_i16_f64(int16_t x) {-  return (double) x;-}--static inline double sitofp_i32_f64(int32_t x) {-  return (double) x;-}--static inline double sitofp_i64_f64(int64_t x) {-  return (double) x;-}--static inline double uitofp_i8_f64(uint8_t x) {-  return (double) x;-}--static inline double uitofp_i16_f64(uint16_t x) {-  return (double) x;-}--static inline double uitofp_i32_f64(uint32_t x) {-  return (double) x;-}--static inline double uitofp_i64_f64(uint64_t x) {-  return (double) x;-}--static inline double fabs64(double x) {-  return fabs(x);-}--static inline double fmax64(double x, double y) {-  return fmax(x, y);-}--static inline double fmin64(double x, double y) {-  return fmin(x, y);-}--static inline double fpow64(double x, double y) {-  return pow(x, y);-}--static inline double futrts_log64(double x) {-  return log(x);-}--static inline double futrts_log2_64(double x) {-  return log2(x);-}--static inline double futrts_log10_64(double x) {-  return log10(x);-}--static inline double futrts_sqrt64(double x) {-  return sqrt(x);-}--static inline double futrts_cbrt64(double x) {-  return cbrt(x);-}--static inline double futrts_exp64(double x) {-  return exp(x);-}--static inline double futrts_cos64(double x) {-  return cos(x);-}--static inline double futrts_sin64(double x) {-  return sin(x);-}--static inline double futrts_tan64(double x) {-  return tan(x);-}--static inline double futrts_acos64(double x) {-  return acos(x);-}--static inline double futrts_asin64(double x) {-  return asin(x);-}--static inline double futrts_atan64(double x) {-  return atan(x);-}--static inline double futrts_cosh64(double x) {-  return cosh(x);-}--static inline double futrts_sinh64(double x) {-  return sinh(x);-}--static inline double futrts_tanh64(double x) {-  return tanh(x);-}--static inline double futrts_acosh64(double x) {-  return acosh(x);-}--static inline double futrts_asinh64(double x) {-  return asinh(x);-}--static inline double futrts_atanh64(double x) {-  return atanh(x);-}--static inline double futrts_atan2_64(double x, double y) {-  return atan2(x, y);-}--static inline double futrts_hypot64(double x, double y) {-  return hypot(x, y);-}--static inline double futrts_gamma64(double x) {-  return tgamma(x);-}--static inline double futrts_lgamma64(double x) {-  return lgamma(x);-}--static inline double futrts_erf64(double x) {-  return erf(x);-}--static inline double futrts_erfc64(double x) {-  return erfc(x);-}--static inline double futrts_fma64(double a, double b, double c) {-  return fma(a, b, c);-}--static inline double futrts_round64(double x) {-  return rint(x);-}--static inline double futrts_ceil64(double x) {-  return ceil(x);-}--static inline double futrts_floor64(double x) {-  return floor(x);-}--static inline bool futrts_isnan64(double x) {-  return isnan(x);-}--static inline bool futrts_isinf64(double x) {-  return isinf(x);-}--static inline int8_t fptosi_f64_i8(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int8_t) x;-  }-}--static inline int16_t fptosi_f64_i16(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int16_t) x;-  }-}--static inline int32_t fptosi_f64_i32(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int32_t) x;-  }-}--static inline int64_t fptosi_f64_i64(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (int64_t) x;-  }-}--static inline uint8_t fptoui_f64_i8(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint8_t) (int8_t) x;-  }-}--static inline uint16_t fptoui_f64_i16(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint16_t) (int16_t) x;-  }-}--static inline uint32_t fptoui_f64_i32(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint32_t) (int32_t) x;-  }-}--static inline uint64_t fptoui_f64_i64(double x) {-  if (futrts_isnan64(x) || futrts_isinf64(x)) {-    return 0;-  } else {-    return (uint64_t) (int64_t) x;-  }-}--static inline bool ftob_f64_bool(double x) {-  return x != 0;-}--static inline double btof_bool_f64(bool x) {-  return x ? 1 : 0;-}--static inline int64_t futrts_to_bits64(double x) {-  union {-    double f;-    int64_t t;-  } p;--  p.f = x;-  return p.t;-}--static inline double futrts_from_bits64(int64_t x) {-  union {-    int64_t f;-    double t;-  } p;--  p.f = x;-  return p.t;-}--static inline double fmod64(double x, double y) {-  return fmod(x, y);-}--static inline double fsignum64(double x) {-  return futrts_isnan64(x) ? x : (x > 0) - (x < 0);-}--static inline double futrts_lerp64(double v0, double v1, double t) {-#ifdef __OPENCL_VERSION__-  return mix(v0, v1, t);-#else-  return v0 + (v1 - v0) * t;-#endif-}--static inline double futrts_mad64(double a, double b, double c) {-#ifdef __OPENCL_VERSION__-  return mad(a, b, c);-#else-  return a * b + c;-#endif-}--static inline float fpconv_f32_f32(float x) {-  return (float) x;-}--static inline double fpconv_f32_f64(float x) {-  return (double) x;-}--static inline float fpconv_f64_f32(double x) {-  return (float) x;-}--static inline double fpconv_f64_f64(double x) {-  return (double) x;-}+#if ISPC++static inline uint8_t udiv8(uint8_t x, uint8_t y) {+  // This strange pattern is used to prevent the ISPC compiler from+  // causing SIGFPEs and bogus results on divisions where inactive lanes+  // have 0-valued divisors. It ensures that any inactive lane instead+  // has a divisor of 1. https://github.com/ispc/ispc/issues/2292+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }++  return x / ys;+}++static inline uint16_t udiv16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x / ys;+}++static inline uint32_t udiv32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  ++  return x / ys;+}++static inline uint64_t udiv64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  ++  return x / ys;+}++static inline uint8_t udiv_up8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  ++  return (x + y - 1) / ys;+}++static inline uint16_t udiv_up16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return (x + y - 1) / ys;+}++static inline uint32_t udiv_up32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return (x + y - 1) / ys;+}++static inline uint64_t udiv_up64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return (x + y - 1) / ys;+}++static inline uint8_t umod8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x % ys;+}++static inline uint16_t umod16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  ++  return x % ys;+}++static inline uint32_t umod32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x % ys;+}++static inline uint64_t umod64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x % ys;+}++static inline uint8_t udiv_safe8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x / ys;+}++static inline uint16_t udiv_safe16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x / ys;+}++static inline uint32_t udiv_safe32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x / ys;+}++static inline uint64_t udiv_safe64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x / ys;+}++static inline uint8_t udiv_up_safe8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : (x + y - 1) / ys;+}++static inline uint16_t udiv_up_safe16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : (x + y - 1) / ys;+}++static inline uint32_t udiv_up_safe32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : (x + y - 1) / ys;+}++static inline uint64_t udiv_up_safe64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : (x + y - 1) / ys;+}++static inline uint8_t umod_safe8(uint8_t x, uint8_t y) {+  uint8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x % ys;+}++static inline uint16_t umod_safe16(uint16_t x, uint16_t y) {+  uint16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x % ys;+}++static inline uint32_t umod_safe32(uint32_t x, uint32_t y) {+  uint32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x % ys;+}++static inline uint64_t umod_safe64(uint64_t x, uint64_t y) {+  uint64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x % ys;+}++static inline int8_t sdiv8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  int8_t q = x / ys;+  int8_t r = x % ys;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++static inline int16_t sdiv16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  int16_t q = x / ys;+  int16_t r = x % ys;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++static inline int32_t sdiv32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  int32_t q = x / ys;+  int32_t r = x % ys;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++static inline int64_t sdiv64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  int64_t q = x / ys;+  int64_t r = x % ys;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++static inline int8_t sdiv_up8(int8_t x, int8_t y) {+  return sdiv8(x + y - 1, y);+}++static inline int16_t sdiv_up16(int16_t x, int16_t y) {+  return sdiv16(x + y - 1, y);+}++static inline int32_t sdiv_up32(int32_t x, int32_t y) {+  return sdiv32(x + y - 1, y);+}++static inline int64_t sdiv_up64(int64_t x, int64_t y) {+  return sdiv64(x + y - 1, y);+}++static inline int8_t smod8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  int8_t r = x % ys;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++static inline int16_t smod16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  int16_t r = x % ys;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++static inline int32_t smod32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  int32_t r = x % ys;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++static inline int64_t smod64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  int64_t r = x % ys;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++static inline int8_t sdiv_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : sdiv8(x, y);+}++static inline int16_t sdiv_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : sdiv16(x, y);+}++static inline int32_t sdiv_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : sdiv32(x, y);+}++static inline int64_t sdiv_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : sdiv64(x, y);+}++static inline int8_t sdiv_up_safe8(int8_t x, int8_t y) {+  return sdiv_safe8(x + y - 1, y);+}++static inline int16_t sdiv_up_safe16(int16_t x, int16_t y) {+  return sdiv_safe16(x + y - 1, y);+}++static inline int32_t sdiv_up_safe32(int32_t x, int32_t y) {+  return sdiv_safe32(x + y - 1, y);+}++static inline int64_t sdiv_up_safe64(int64_t x, int64_t y) {+  return sdiv_safe64(x + y - 1, y);+}++static inline int8_t smod_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : smod8(x, y);+}++static inline int16_t smod_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : smod16(x, y);+}++static inline int32_t smod_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : smod32(x, y);+}++static inline int64_t smod_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : smod64(x, y);+}++static inline int8_t squot8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x / ys;+}++static inline int16_t squot16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x / ys;+}++static inline int32_t squot32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x / ys;+}++static inline int64_t squot64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x / ys;+}++static inline int8_t srem8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x % ys;+}++static inline int16_t srem16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x % ys;+}++static inline int32_t srem32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x % ys;+}++static inline int64_t srem64(int64_t x, int64_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return x % ys;+}++static inline int8_t squot_safe8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x / ys;+}++static inline int16_t squot_safe16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x / ys;+}++static inline int32_t squot_safe32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x / ys;+}++static inline int64_t squot_safe64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x / ys;+}++static inline int8_t srem_safe8(int8_t x, int8_t y) {+  int8_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x % ys;+}++static inline int16_t srem_safe16(int16_t x, int16_t y) {+  int16_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x % ys;+}++static inline int32_t srem_safe32(int32_t x, int32_t y) {+  int32_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x % ys;+}++static inline int64_t srem_safe64(int64_t x, int64_t y) {+  int64_t ys = 1;+  foreach_active(i){+    ys = y;+  }+  +  return y == 0 ? 0 : x % ys;+}++#else++static inline uint8_t udiv8(uint8_t x, uint8_t y) {+  return x / y;+}++static inline uint16_t udiv16(uint16_t x, uint16_t y) {+  return x / y;+}++static inline uint32_t udiv32(uint32_t x, uint32_t y) {+  return x / y;+}++static inline uint64_t udiv64(uint64_t x, uint64_t y) {+  return x / y;+}++static inline uint8_t udiv_up8(uint8_t x, uint8_t y) {+  return (x + y - 1) / y;+}++static inline uint16_t udiv_up16(uint16_t x, uint16_t y) {+  return (x + y - 1) / y;+}++static inline uint32_t udiv_up32(uint32_t x, uint32_t y) {+  return (x + y - 1) / y;+}++static inline uint64_t udiv_up64(uint64_t x, uint64_t y) {+  return (x + y - 1) / y;+}++static inline uint8_t umod8(uint8_t x, uint8_t y) {+  return x % y;+}++static inline uint16_t umod16(uint16_t x, uint16_t y) {+  return x % y;+}++static inline uint32_t umod32(uint32_t x, uint32_t y) {+  return x % y;+}++static inline uint64_t umod64(uint64_t x, uint64_t y) {+  return x % y;+}++static inline uint8_t udiv_safe8(uint8_t x, uint8_t y) {+  return y == 0 ? 0 : x / y;+}++static inline uint16_t udiv_safe16(uint16_t x, uint16_t y) {+  return y == 0 ? 0 : x / y;+}++static inline uint32_t udiv_safe32(uint32_t x, uint32_t y) {+  return y == 0 ? 0 : x / y;+}++static inline uint64_t udiv_safe64(uint64_t x, uint64_t y) {+  return y == 0 ? 0 : x / y;+}++static inline uint8_t udiv_up_safe8(uint8_t x, uint8_t y) {+  return y == 0 ? 0 : (x + y - 1) / y;+}++static inline uint16_t udiv_up_safe16(uint16_t x, uint16_t y) {+  return y == 0 ? 0 : (x + y - 1) / y;+}++static inline uint32_t udiv_up_safe32(uint32_t x, uint32_t y) {+  return y == 0 ? 0 : (x + y - 1) / y;+}++static inline uint64_t udiv_up_safe64(uint64_t x, uint64_t y) {+  return y == 0 ? 0 : (x + y - 1) / y;+}++static inline uint8_t umod_safe8(uint8_t x, uint8_t y) {+  return y == 0 ? 0 : x % y;+}++static inline uint16_t umod_safe16(uint16_t x, uint16_t y) {+  return y == 0 ? 0 : x % y;+}++static inline uint32_t umod_safe32(uint32_t x, uint32_t y) {+  return y == 0 ? 0 : x % y;+}++static inline uint64_t umod_safe64(uint64_t x, uint64_t y) {+  return y == 0 ? 0 : x % y;+}++static inline int8_t sdiv8(int8_t x, int8_t y) {+  int8_t q = x / y;+  int8_t r = x % y;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++static inline int16_t sdiv16(int16_t x, int16_t y) {+  int16_t q = x / y;+  int16_t r = x % y;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++static inline int32_t sdiv32(int32_t x, int32_t y) {+  int32_t q = x / y;+  int32_t r = x % y;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++static inline int64_t sdiv64(int64_t x, int64_t y) {+  int64_t q = x / y;+  int64_t r = x % y;++  return q - ((r != 0 && r < 0 != y < 0) ? 1 : 0);+}++static inline int8_t sdiv_up8(int8_t x, int8_t y) {+  return sdiv8(x + y - 1, y);+}++static inline int16_t sdiv_up16(int16_t x, int16_t y) {+  return sdiv16(x + y - 1, y);+}++static inline int32_t sdiv_up32(int32_t x, int32_t y) {+  return sdiv32(x + y - 1, y);+}++static inline int64_t sdiv_up64(int64_t x, int64_t y) {+  return sdiv64(x + y - 1, y);+}++static inline int8_t smod8(int8_t x, int8_t y) {+  int8_t r = x % y;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++static inline int16_t smod16(int16_t x, int16_t y) {+  int16_t r = x % y;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++static inline int32_t smod32(int32_t x, int32_t y) {+  int32_t r = x % y;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++static inline int64_t smod64(int64_t x, int64_t y) {+  int64_t r = x % y;++  return r + (r == 0 || (x > 0 && y > 0) || (x < 0 && y < 0) ? 0 : y);+}++static inline int8_t sdiv_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : sdiv8(x, y);+}++static inline int16_t sdiv_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : sdiv16(x, y);+}++static inline int32_t sdiv_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : sdiv32(x, y);+}++static inline int64_t sdiv_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : sdiv64(x, y);+}++static inline int8_t sdiv_up_safe8(int8_t x, int8_t y) {+  return sdiv_safe8(x + y - 1, y);+}++static inline int16_t sdiv_up_safe16(int16_t x, int16_t y) {+  return sdiv_safe16(x + y - 1, y);+}++static inline int32_t sdiv_up_safe32(int32_t x, int32_t y) {+  return sdiv_safe32(x + y - 1, y);+}++static inline int64_t sdiv_up_safe64(int64_t x, int64_t y) {+  return sdiv_safe64(x + y - 1, y);+}++static inline int8_t smod_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : smod8(x, y);+}++static inline int16_t smod_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : smod16(x, y);+}++static inline int32_t smod_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : smod32(x, y);+}++static inline int64_t smod_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : smod64(x, y);+}++static inline int8_t squot8(int8_t x, int8_t y) {+  return x / y;+}++static inline int16_t squot16(int16_t x, int16_t y) {+  return x / y;+}++static inline int32_t squot32(int32_t x, int32_t y) {+  return x / y;+}++static inline int64_t squot64(int64_t x, int64_t y) {+  return x / y;+}++static inline int8_t srem8(int8_t x, int8_t y) {+  return x % y;+}++static inline int16_t srem16(int16_t x, int16_t y) {+  return x % y;+}++static inline int32_t srem32(int32_t x, int32_t y) {+  return x % y;+}++static inline int64_t srem64(int64_t x, int64_t y) {+  return x % y;+}++static inline int8_t squot_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : x / y;+}++static inline int16_t squot_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : x / y;+}++static inline int32_t squot_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : x / y;+}++static inline int64_t squot_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : x / y;+}++static inline int8_t srem_safe8(int8_t x, int8_t y) {+  return y == 0 ? 0 : x % y;+}++static inline int16_t srem_safe16(int16_t x, int16_t y) {+  return y == 0 ? 0 : x % y;+}++static inline int32_t srem_safe32(int32_t x, int32_t y) {+  return y == 0 ? 0 : x % y;+}++static inline int64_t srem_safe64(int64_t x, int64_t y) {+  return y == 0 ? 0 : x % y;+}++#endif++static inline int8_t smin8(int8_t x, int8_t y) {+  return x < y ? x : y;+}++static inline int16_t smin16(int16_t x, int16_t y) {+  return x < y ? x : y;+}++static inline int32_t smin32(int32_t x, int32_t y) {+  return x < y ? x : y;+}++static inline int64_t smin64(int64_t x, int64_t y) {+  return x < y ? x : y;+}++static inline uint8_t umin8(uint8_t x, uint8_t y) {+  return x < y ? x : y;+}++static inline uint16_t umin16(uint16_t x, uint16_t y) {+  return x < y ? x : y;+}++static inline uint32_t umin32(uint32_t x, uint32_t y) {+  return x < y ? x : y;+}++static inline uint64_t umin64(uint64_t x, uint64_t y) {+  return x < y ? x : y;+}++static inline int8_t smax8(int8_t x, int8_t y) {+  return x < y ? y : x;+}++static inline int16_t smax16(int16_t x, int16_t y) {+  return x < y ? y : x;+}++static inline int32_t smax32(int32_t x, int32_t y) {+  return x < y ? y : x;+}++static inline int64_t smax64(int64_t x, int64_t y) {+  return x < y ? y : x;+}++static inline uint8_t umax8(uint8_t x, uint8_t y) {+  return x < y ? y : x;+}++static inline uint16_t umax16(uint16_t x, uint16_t y) {+  return x < y ? y : x;+}++static inline uint32_t umax32(uint32_t x, uint32_t y) {+  return x < y ? y : x;+}++static inline uint64_t umax64(uint64_t x, uint64_t y) {+  return x < y ? y : x;+}++static inline uint8_t shl8(uint8_t x, uint8_t y) {+  return (uint8_t)(x << y);+}++static inline uint16_t shl16(uint16_t x, uint16_t y) {+  return (uint16_t)(x << y);+}++static inline uint32_t shl32(uint32_t x, uint32_t y) {+  return x << y;+}++static inline uint64_t shl64(uint64_t x, uint64_t y) {+  return x << y;+}++static inline uint8_t lshr8(uint8_t x, uint8_t y) {+  return x >> y;+}++static inline uint16_t lshr16(uint16_t x, uint16_t y) {+  return x >> y;+}++static inline uint32_t lshr32(uint32_t x, uint32_t y) {+  return x >> y;+}++static inline uint64_t lshr64(uint64_t x, uint64_t y) {+  return x >> y;+}++static inline int8_t ashr8(int8_t x, int8_t y) {+  return x >> y;+}++static inline int16_t ashr16(int16_t x, int16_t y) {+  return x >> y;+}++static inline int32_t ashr32(int32_t x, int32_t y) {+  return x >> y;+}++static inline int64_t ashr64(int64_t x, int64_t y) {+  return x >> y;+}++static inline uint8_t and8(uint8_t x, uint8_t y) {+  return x & y;+}++static inline uint16_t and16(uint16_t x, uint16_t y) {+  return x & y;+}++static inline uint32_t and32(uint32_t x, uint32_t y) {+  return x & y;+}++static inline uint64_t and64(uint64_t x, uint64_t y) {+  return x & y;+}++static inline uint8_t or8(uint8_t x, uint8_t y) {+  return x | y;+}++static inline uint16_t or16(uint16_t x, uint16_t y) {+  return x | y;+}++static inline uint32_t or32(uint32_t x, uint32_t y) {+  return x | y;+}++static inline uint64_t or64(uint64_t x, uint64_t y) {+  return x | y;+}++static inline uint8_t xor8(uint8_t x, uint8_t y) {+  return x ^ y;+}++static inline uint16_t xor16(uint16_t x, uint16_t y) {+  return x ^ y;+}++static inline uint32_t xor32(uint32_t x, uint32_t y) {+  return x ^ y;+}++static inline uint64_t xor64(uint64_t x, uint64_t y) {+  return x ^ y;+}++static inline bool ult8(uint8_t x, uint8_t y) {+  return x < y;+}++static inline bool ult16(uint16_t x, uint16_t y) {+  return x < y;+}++static inline bool ult32(uint32_t x, uint32_t y) {+  return x < y;+}++static inline bool ult64(uint64_t x, uint64_t y) {+  return x < y;+}++static inline bool ule8(uint8_t x, uint8_t y) {+  return x <= y;+}++static inline bool ule16(uint16_t x, uint16_t y) {+  return x <= y;+}++static inline bool ule32(uint32_t x, uint32_t y) {+  return x <= y;+}++static inline bool ule64(uint64_t x, uint64_t y) {+  return x <= y;+}++static inline bool slt8(int8_t x, int8_t y) {+  return x < y;+}++static inline bool slt16(int16_t x, int16_t y) {+  return x < y;+}++static inline bool slt32(int32_t x, int32_t y) {+  return x < y;+}++static inline bool slt64(int64_t x, int64_t y) {+  return x < y;+}++static inline bool sle8(int8_t x, int8_t y) {+  return x <= y;+}++static inline bool sle16(int16_t x, int16_t y) {+  return x <= y;+}++static inline bool sle32(int32_t x, int32_t y) {+  return x <= y;+}++static inline bool sle64(int64_t x, int64_t y) {+  return x <= y;+}++static inline uint8_t pow8(uint8_t x, uint8_t y) {+  uint8_t res = 1, rem = y;++  while (rem != 0) {+    if (rem & 1)+      res *= x;+    rem >>= 1;+    x *= x;+  }+  return res;+}++static inline uint16_t pow16(uint16_t x, uint16_t y) {+  uint16_t res = 1, rem = y;++  while (rem != 0) {+    if (rem & 1)+      res *= x;+    rem >>= 1;+    x *= x;+  }+  return res;+}++static inline uint32_t pow32(uint32_t x, uint32_t y) {+  uint32_t res = 1, rem = y;++  while (rem != 0) {+    if (rem & 1)+      res *= x;+    rem >>= 1;+    x *= x;+  }+  return res;+}++static inline uint64_t pow64(uint64_t x, uint64_t y) {+  uint64_t res = 1, rem = y;++  while (rem != 0) {+    if (rem & 1)+      res *= x;+    rem >>= 1;+    x *= x;+  }+  return res;+}++static inline bool itob_i8_bool(int8_t x) {+  return x != 0;+}++static inline bool itob_i16_bool(int16_t x) {+  return x != 0;+}++static inline bool itob_i32_bool(int32_t x) {+  return x != 0;+}++static inline bool itob_i64_bool(int64_t x) {+  return x != 0;+}++static inline int8_t btoi_bool_i8(bool x) {+  return x;+}++static inline int16_t btoi_bool_i16(bool x) {+  return x;+}++static inline int32_t btoi_bool_i32(bool x) {+  return x;+}++static inline int64_t btoi_bool_i64(bool x) {+  return x;+}++#define sext_i8_i8(x) ((int8_t) (int8_t) (x))+#define sext_i8_i16(x) ((int16_t) (int8_t) (x))+#define sext_i8_i32(x) ((int32_t) (int8_t) (x))+#define sext_i8_i64(x) ((int64_t) (int8_t) (x))+#define sext_i16_i8(x) ((int8_t) (int16_t) (x))+#define sext_i16_i16(x) ((int16_t) (int16_t) (x))+#define sext_i16_i32(x) ((int32_t) (int16_t) (x))+#define sext_i16_i64(x) ((int64_t) (int16_t) (x))+#define sext_i32_i8(x) ((int8_t) (int32_t) (x))+#define sext_i32_i16(x) ((int16_t) (int32_t) (x))+#define sext_i32_i32(x) ((int32_t) (int32_t) (x))+#define sext_i32_i64(x) ((int64_t) (int32_t) (x))+#define sext_i64_i8(x) ((int8_t) (int64_t) (x))+#define sext_i64_i16(x) ((int16_t) (int64_t) (x))+#define sext_i64_i32(x) ((int32_t) (int64_t) (x))+#define sext_i64_i64(x) ((int64_t) (int64_t) (x))+#define zext_i8_i8(x) ((int8_t) (uint8_t) (x))+#define zext_i8_i16(x) ((int16_t) (uint8_t) (x))+#define zext_i8_i32(x) ((int32_t) (uint8_t) (x))+#define zext_i8_i64(x) ((int64_t) (uint8_t) (x))+#define zext_i16_i8(x) ((int8_t) (uint16_t) (x))+#define zext_i16_i16(x) ((int16_t) (uint16_t) (x))+#define zext_i16_i32(x) ((int32_t) (uint16_t) (x))+#define zext_i16_i64(x) ((int64_t) (uint16_t) (x))+#define zext_i32_i8(x) ((int8_t) (uint32_t) (x))+#define zext_i32_i16(x) ((int16_t) (uint32_t) (x))+#define zext_i32_i32(x) ((int32_t) (uint32_t) (x))+#define zext_i32_i64(x) ((int64_t) (uint32_t) (x))+#define zext_i64_i8(x) ((int8_t) (uint64_t) (x))+#define zext_i64_i16(x) ((int16_t) (uint64_t) (x))+#define zext_i64_i32(x) ((int32_t) (uint64_t) (x))+#define zext_i64_i64(x) ((int64_t) (uint64_t) (x))++static int8_t abs8(int8_t x) {+  return (int8_t)abs(x);+}++static int16_t abs16(int16_t x) {+  return (int16_t)abs(x);+}++static int32_t abs32(int32_t x) {+  return abs(x);+}++static int64_t abs64(int64_t x) {+#if defined(__OPENCL_VERSION__) || defined(ISPC)+  return abs(x);+#else+  return llabs(x);+#endif+}++#if defined(__OPENCL_VERSION__)+static int32_t futrts_popc8(int8_t x) {+  return popcount(x);+}++static int32_t futrts_popc16(int16_t x) {+  return popcount(x);+}++static int32_t futrts_popc32(int32_t x) {+  return popcount(x);+}++static int32_t futrts_popc64(int64_t x) {+  return popcount(x);+}+#elif defined(__CUDA_ARCH__)++static int32_t futrts_popc8(int8_t x) {+  return __popc(zext_i8_i32(x));+}++static int32_t futrts_popc16(int16_t x) {+  return __popc(zext_i16_i32(x));+}++static int32_t futrts_popc32(int32_t x) {+  return __popc(x);+}++static int32_t futrts_popc64(int64_t x) {+  return __popcll(x);+}++#else // Not OpenCL or CUDA, but plain C.++static int32_t futrts_popc8(uint8_t x) {+  int c = 0;+  for (; x; ++c) { x &= x - 1; }+  return c;+}++static int32_t futrts_popc16(uint16_t x) {+  int c = 0;+  for (; x; ++c) { x &= x - 1; }+  return c;+}++static int32_t futrts_popc32(uint32_t x) {+  int c = 0;+  for (; x; ++c) { x &= x - 1; }+  return c;+}++static int32_t futrts_popc64(uint64_t x) {+  int c = 0;+  for (; x; ++c) { x &= x - 1; }+  return c;+}+#endif++#if defined(__OPENCL_VERSION__)+static uint8_t futrts_mul_hi8(uint8_t a, uint8_t b) {+  return mul_hi(a, b);+}++static uint16_t futrts_mul_hi16(uint16_t a, uint16_t b) {+  return mul_hi(a, b);+}++static uint32_t futrts_mul_hi32(uint32_t a, uint32_t b) {+  return mul_hi(a, b);+}++static uint64_t futrts_mul_hi64(uint64_t a, uint64_t b) {+  return mul_hi(a, b);+}++#elif defined(__CUDA_ARCH__)++static uint8_t futrts_mul_hi8(uint8_t a, uint8_t b) {+  uint16_t aa = a;+  uint16_t bb = b;++  return aa * bb >> 8;+}++static uint16_t futrts_mul_hi16(uint16_t a, uint16_t b) {+  uint32_t aa = a;+  uint32_t bb = b;++  return aa * bb >> 16;+}++static uint32_t futrts_mul_hi32(uint32_t a, uint32_t b) {+  return mulhi(a, b);+}++static uint64_t futrts_mul_hi64(uint64_t a, uint64_t b) {+  return mul64hi(a, b);+}++#elif ISPC++static uint8_t futrts_mul_hi8(uint8_t a, uint8_t b) {+  uint16_t aa = a;+  uint16_t bb = b;++  return aa * bb >> 8;+}++static uint16_t futrts_mul_hi16(uint16_t a, uint16_t b) {+  uint32_t aa = a;+  uint32_t bb = b;++  return aa * bb >> 16;+}++static uint32_t futrts_mul_hi32(uint32_t a, uint32_t b) {+  uint64_t aa = a;+  uint64_t bb = b;++  return aa * bb >> 32;+}++static uint64_t futrts_mul_hi64(uint64_t a, uint64_t b) {+  uint64_t ah = a >> 32;+  uint64_t al = a & 0xffffffff;+  uint64_t bh = b >> 32;+  uint64_t bl = b & 0xffffffff;++  uint64_t p1 = al * bl;+  uint64_t p2 = al * bh;+  uint64_t p3 = ah * bl;+  uint64_t p4 = ah * bh;++  uint64_t p1h = p1 >> 32;+  uint64_t p2h = p2 >> 32;+  uint64_t p3h = p3 >> 32;+  uint64_t p2l = p2 & 0xffffffff;+  uint64_t p3l = p3 & 0xffffffff;++  uint64_t l = p1h + p2l  + p3l;+  uint64_t m = (p2 >> 32) + (p3 >> 32);+  uint64_t h = (l >> 32) + m + p4;++  return h;+}++#else // Not OpenCL, ISPC, or CUDA, but plain C.++static uint8_t futrts_mul_hi8(uint8_t a, uint8_t b) {+  uint16_t aa = a;+  uint16_t bb = b;++  return aa * bb >> 8;+}++static uint16_t futrts_mul_hi16(uint16_t a, uint16_t b) {+  uint32_t aa = a;+  uint32_t bb = b;++  return aa * bb >> 16;+}++static uint32_t futrts_mul_hi32(uint32_t a, uint32_t b) {+  uint64_t aa = a;+  uint64_t bb = b;++  return aa * bb >> 32;+}++static uint64_t futrts_mul_hi64(uint64_t a, uint64_t b) {+  __uint128_t aa = a;+  __uint128_t bb = b;++  return aa * bb >> 64;+}+#endif++#if defined(__OPENCL_VERSION__)+static uint8_t futrts_mad_hi8(uint8_t a, uint8_t b, uint8_t c) {+  return mad_hi(a, b, c);+}++static uint16_t futrts_mad_hi16(uint16_t a, uint16_t b, uint16_t c) {+  return mad_hi(a, b, c);+}++static uint32_t futrts_mad_hi32(uint32_t a, uint32_t b, uint32_t c) {+  return mad_hi(a, b, c);+}++static uint64_t futrts_mad_hi64(uint64_t a, uint64_t b, uint64_t c) {+  return mad_hi(a, b, c);+}++#else // Not OpenCL++static uint8_t futrts_mad_hi8(uint8_t a, uint8_t b, uint8_t c) {+  return futrts_mul_hi8(a, b) + c;+}++static uint16_t futrts_mad_hi16(uint16_t a, uint16_t b, uint16_t c) {+  return futrts_mul_hi16(a, b) + c;+}++static uint32_t futrts_mad_hi32(uint32_t a, uint32_t b, uint32_t c) {+  return futrts_mul_hi32(a, b) + c;+}++static uint64_t futrts_mad_hi64(uint64_t a, uint64_t b, uint64_t c) {+  return futrts_mul_hi64(a, b) + c;+}+#endif++#if defined(__OPENCL_VERSION__)+static int32_t futrts_clzz8(int8_t x) {+  return clz(x);+}++static int32_t futrts_clzz16(int16_t x) {+  return clz(x);+}++static int32_t futrts_clzz32(int32_t x) {+  return clz(x);+}++static int32_t futrts_clzz64(int64_t x) {+  return clz(x);+}++#elif defined(__CUDA_ARCH__)++static int32_t futrts_clzz8(int8_t x) {+  return __clz(zext_i8_i32(x)) - 24;+}++static int32_t futrts_clzz16(int16_t x) {+  return __clz(zext_i16_i32(x)) - 16;+}++static int32_t futrts_clzz32(int32_t x) {+  return __clz(x);+}++static int32_t futrts_clzz64(int64_t x) {+  return __clzll(x);+}++#elif ISPC++static int32_t futrts_clzz8(int8_t x) {+  return count_leading_zeros((int32_t)(uint8_t)x)-24;+}++static int32_t futrts_clzz16(int16_t x) {+  return count_leading_zeros((int32_t)(uint16_t)x)-16;+}++static int32_t futrts_clzz32(int32_t x) {+  return count_leading_zeros(x);+}++static int32_t futrts_clzz64(int64_t x) {+  return count_leading_zeros(x);+}++#else // Not OpenCL, ISPC or CUDA, but plain C.++static int32_t futrts_clzz8(int8_t x) {+  return x == 0 ? 8 : __builtin_clz((uint32_t)zext_i8_i32(x)) - 24;+}++static int32_t futrts_clzz16(int16_t x) {+  return x == 0 ? 16 : __builtin_clz((uint32_t)zext_i16_i32(x)) - 16;+}++static int32_t futrts_clzz32(int32_t x) {+  return x == 0 ? 32 : __builtin_clz((uint32_t)x);+}++static int32_t futrts_clzz64(int64_t x) {+  return x == 0 ? 64 : __builtin_clzll((uint64_t)x);+}+#endif++#if defined(__OPENCL_VERSION__)+static int32_t futrts_ctzz8(int8_t x) {+  int i = 0;+  for (; i < 8 && (x & 1) == 0; i++, x >>= 1)+    ;+  return i;+}++static int32_t futrts_ctzz16(int16_t x) {+  int i = 0;+  for (; i < 16 && (x & 1) == 0; i++, x >>= 1)+    ;+  return i;+}++static int32_t futrts_ctzz32(int32_t x) {+  int i = 0;+  for (; i < 32 && (x & 1) == 0; i++, x >>= 1)+    ;+  return i;+}++static int32_t futrts_ctzz64(int64_t x) {+  int i = 0;+  for (; i < 64 && (x & 1) == 0; i++, x >>= 1)+    ;+  return i;+}++#elif defined(__CUDA_ARCH__)++static int32_t futrts_ctzz8(int8_t x) {+  int y = __ffs(x);+  return y == 0 ? 8 : y - 1;+}++static int32_t futrts_ctzz16(int16_t x) {+  int y = __ffs(x);+  return y == 0 ? 16 : y - 1;+}++static int32_t futrts_ctzz32(int32_t x) {+  int y = __ffs(x);+  return y == 0 ? 32 : y - 1;+}++static int32_t futrts_ctzz64(int64_t x) {+  int y = __ffsll(x);+  return y == 0 ? 64 : y - 1;+}++#elif ISPC++static int32_t futrts_ctzz8(int8_t x) {+  return x == 0 ? 8 : count_trailing_zeros((int32_t)x);+}++static int32_t futrts_ctzz16(int16_t x) {+  return x == 0 ? 16 : count_trailing_zeros((int32_t)x);+}++static int32_t futrts_ctzz32(int32_t x) {+  return count_trailing_zeros(x);+}++static int32_t futrts_ctzz64(int64_t x) {+  return count_trailing_zeros(x);+}++#else // Not OpenCL or CUDA, but plain C.++static int32_t futrts_ctzz8(int8_t x) {+  return x == 0 ? 8 : __builtin_ctz((uint32_t)x);+}++static int32_t futrts_ctzz16(int16_t x) {+  return x == 0 ? 16 : __builtin_ctz((uint32_t)x);+}++static int32_t futrts_ctzz32(int32_t x) {+  return x == 0 ? 32 : __builtin_ctz((uint32_t)x);+}++static int32_t futrts_ctzz64(int64_t x) {+  return x == 0 ? 64 : __builtin_ctzll((uint64_t)x);+}+#endif++static inline float fdiv32(float x, float y) {+  return x / y;+}++static inline float fadd32(float x, float y) {+  return x + y;+}++static inline float fsub32(float x, float y) {+  return x - y;+}++static inline float fmul32(float x, float y) {+  return x * y;+}++static inline bool cmplt32(float x, float y) {+  return x < y;+}++static inline bool cmple32(float x, float y) {+  return x <= y;+}++static inline float sitofp_i8_f32(int8_t x) {+  return (float) x;+}++static inline float sitofp_i16_f32(int16_t x) {+  return (float) x;+}++static inline float sitofp_i32_f32(int32_t x) {+  return (float) x;+}++static inline float sitofp_i64_f32(int64_t x) {+  return (float) x;+}++static inline float uitofp_i8_f32(uint8_t x) {+  return (float) x;+}++static inline float uitofp_i16_f32(uint16_t x) {+  return (float) x;+}++static inline float uitofp_i32_f32(uint32_t x) {+  return (float) x;+}++static inline float uitofp_i64_f32(uint64_t x) {+  return (float) x;+}++#ifdef __OPENCL_VERSION__+static inline float fabs32(float x) {+  return fabs(x);+}++static inline float fmax32(float x, float y) {+  return fmax(x, y);+}++static inline float fmin32(float x, float y) {+  return fmin(x, y);+}++static inline float fpow32(float x, float y) {+  return pow(x, y);+}++#elif ISPC++static inline float fabs32(float x) {+  return abs(x);+}++static inline float fmax32(float x, float y) {+  return isnan(x) ? y : isnan(y) ? x : max(x, y);+}++static inline float fmin32(float x, float y) {+  return isnan(x) ? y : isnan(y) ? x : min(x, y);+}++static inline float fpow32(float a, float b) {+  float ret;+  foreach_active (i) {+      uniform float r = __stdlib_powf(extract(a, i), extract(b, i));+      ret = insert(ret, i, r);+  }+  return ret;+}++#else // Not OpenCL, but CUDA or plain C.++static inline float fabs32(float x) {+  return fabsf(x);+}++static inline float fmax32(float x, float y) {+  return fmaxf(x, y);+}++static inline float fmin32(float x, float y) {+  return fminf(x, y);+}++static inline float fpow32(float x, float y) {+  return powf(x, y);+}+#endif++static inline bool futrts_isnan32(float x) {+  return isnan(x);+}++#if ISPC++static inline bool futrts_isinf32(float x) {+  return !isnan(x) && isnan(x - x);+}++static inline bool futrts_isfinite32(float x) {+  return !isnan(x) && !futrts_isinf32(x);+}++#else++static inline bool futrts_isinf32(float x) {+  return isinf(x);+}++#endif++static inline int8_t fptosi_f32_i8(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (int8_t) x;+  }+}++static inline int16_t fptosi_f32_i16(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (int16_t) x;+  }+}++static inline int32_t fptosi_f32_i32(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (int32_t) x;+  }+}++static inline int64_t fptosi_f32_i64(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (int64_t) x;+  };+}++static inline uint8_t fptoui_f32_i8(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (uint8_t) (int8_t) x;+  }+}++static inline uint16_t fptoui_f32_i16(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (uint16_t) (int16_t) x;+  }+}++static inline uint32_t fptoui_f32_i32(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (uint32_t) (int32_t) x;+  }+}++static inline uint64_t fptoui_f32_i64(float x) {+  if (futrts_isnan32(x) || futrts_isinf32(x)) {+    return 0;+  } else {+    return (uint64_t) (int64_t) x;+  }+}++static inline bool ftob_f32_bool(float x) {+  return x != 0;+}++static inline float btof_bool_f32(bool x) {+  return x ? 1 : 0;+}++#ifdef __OPENCL_VERSION__+static inline float futrts_log32(float x) {+  return log(x);+}++static inline float futrts_log2_32(float x) {+  return log2(x);+}++static inline float futrts_log10_32(float x) {+  return log10(x);+}++static inline float futrts_sqrt32(float x) {+  return sqrt(x);+}++static inline float futrts_cbrt32(float x) {+  return cbrt(x);+}++static inline float futrts_exp32(float x) {+  return exp(x);+}++static inline float futrts_cos32(float x) {+  return cos(x);+}++static inline float futrts_sin32(float x) {+  return sin(x);+}++static inline float futrts_tan32(float x) {+  return tan(x);+}++static inline float futrts_acos32(float x) {+  return acos(x);+}++static inline float futrts_asin32(float x) {+  return asin(x);+}++static inline float futrts_atan32(float x) {+  return atan(x);+}++static inline float futrts_cosh32(float x) {+  return cosh(x);+}++static inline float futrts_sinh32(float x) {+  return sinh(x);+}++static inline float futrts_tanh32(float x) {+  return tanh(x);+}++static inline float futrts_acosh32(float x) {+  return acosh(x);+}++static inline float futrts_asinh32(float x) {+  return asinh(x);+}++static inline float futrts_atanh32(float x) {+  return atanh(x);+}++static inline float futrts_atan2_32(float x, float y) {+  return atan2(x, y);+}++static inline float futrts_hypot32(float x, float y) {+  return hypot(x, y);+}++static inline float futrts_gamma32(float x) {+  return tgamma(x);+}++static inline float futrts_lgamma32(float x) {+  return lgamma(x);+}++static inline float futrts_erf32(float x) {+  return erf(x);+}++static inline float futrts_erfc32(float x) {+  return erfc(x);+}++static inline float fmod32(float x, float y) {+  return fmod(x, y);+}++static inline float futrts_round32(float x) {+  return rint(x);+}++static inline float futrts_floor32(float x) {+  return floor(x);+}++static inline float futrts_ceil32(float x) {+  return ceil(x);+}++static inline float futrts_lerp32(float v0, float v1, float t) {+  return mix(v0, v1, t);+}++static inline float futrts_mad32(float a, float b, float c) {+  return mad(a, b, c);+}++static inline float futrts_fma32(float a, float b, float c) {+  return fma(a, b, c);+}++#elif ISPC++static inline float futrts_log32(float x) {+  return futrts_isfinite32(x) || (futrts_isinf32(x) && x < 0)? log(x) : x;+}++static inline float futrts_log2_32(float x) {+  return futrts_log32(x) / log(2.0f);+}++static inline float futrts_log10_32(float x) {+  return futrts_log32(x) / log(10.0f);+}++static inline float futrts_sqrt32(float x) {+  return sqrt(x);+}++extern "C" unmasked uniform float cbrtf(uniform float);+static inline float futrts_cbrt32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = cbrtf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++static inline float futrts_exp32(float x) {+  return exp(x);+}++static inline float futrts_cos32(float x) {+  return cos(x);+}++static inline float futrts_sin32(float x) {+  return sin(x);+}++static inline float futrts_tan32(float x) {+  return tan(x);+}++static inline float futrts_acos32(float x) {+  return acos(x);+}++static inline float futrts_asin32(float x) {+  return asin(x);+}++static inline float futrts_atan32(float x) {+  return atan(x);+}++static inline float futrts_cosh32(float x) {+  return (exp(x)+exp(-x)) / 2.0f;+}++static inline float futrts_sinh32(float x) {+  return (exp(x)-exp(-x)) / 2.0f;+}++static inline float futrts_tanh32(float x) {+  return futrts_sinh32(x)/futrts_cosh32(x);+}++static inline float futrts_acosh32(float x) {+  float f = x+sqrt(x*x-1);+  if(futrts_isfinite32(f)) return log(f);+  return f;+}++static inline float futrts_asinh32(float x) {+  float f = x+sqrt(x*x+1);+  if(futrts_isfinite32(f)) return log(f);+  return f;++}++static inline float futrts_atanh32(float x) {+  float f = (1+x)/(1-x);+  if(futrts_isfinite32(f)) return log(f)/2.0f;+  return f;++}++static inline float futrts_atan2_32(float x, float y) {+  return (x == 0.0f && y == 0.0f) ? 0.0f : atan2(x, y);+}++static inline float futrts_hypot32(float x, float y) {+  if (futrts_isfinite32(x) && futrts_isfinite32(y)) {+    x = abs(x);+    y = abs(y);+    float a;+    float b;+    if (x >= y){+        a = x;+        b = y;+    } else {+        a = y;+        b = x;+    }+    if(b == 0){+      return a;+    }++    int e;+    float an;+    float bn;+    an = frexp (a, &e);+    bn = ldexp (b, - e);+    float cn;+    cn = sqrt (an * an + bn * bn);+    return ldexp (cn, e);+  } else {+    if (futrts_isinf32(x) || futrts_isinf32(y)) return INFINITY;+    else return x + y;+  }++}++extern "C" unmasked uniform float tgammaf(uniform float x);+static inline float futrts_gamma32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = tgammaf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform float lgammaf(uniform float x);+static inline float futrts_lgamma32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = lgammaf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform float erff(uniform float x);+static inline float futrts_erf32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = erff(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform float erfcf(uniform float x);+static inline float futrts_erfc32(float x) {+  float res;+  foreach_active (i) {+    uniform float r = erfcf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++static inline float fmod32(float x, float y) {+  return x - y * trunc(x/y);+}++static inline float futrts_round32(float x) {+  return round(x);+}++static inline float futrts_floor32(float x) {+  return floor(x);+}++static inline float futrts_ceil32(float x) {+  return ceil(x);+}++static inline float futrts_lerp32(float v0, float v1, float t) {+  return v0 + (v1 - v0) * t;+}++static inline float futrts_mad32(float a, float b, float c) {+  return a * b + c;+}++static inline float futrts_fma32(float a, float b, float c) {+  return a * b + c;+}++#else // Not OpenCL or ISPC, but CUDA or plain C.++static inline float futrts_log32(float x) {+  return logf(x);+}++static inline float futrts_log2_32(float x) {+  return log2f(x);+}++static inline float futrts_log10_32(float x) {+  return log10f(x);+}++static inline float futrts_sqrt32(float x) {+  return sqrtf(x);+}++static inline float futrts_cbrt32(float x) {+  return cbrtf(x);+}++static inline float futrts_exp32(float x) {+  return expf(x);+}++static inline float futrts_cos32(float x) {+  return cosf(x);+}++static inline float futrts_sin32(float x) {+  return sinf(x);+}++static inline float futrts_tan32(float x) {+  return tanf(x);+}++static inline float futrts_acos32(float x) {+  return acosf(x);+}++static inline float futrts_asin32(float x) {+  return asinf(x);+}++static inline float futrts_atan32(float x) {+  return atanf(x);+}++static inline float futrts_cosh32(float x) {+  return coshf(x);+}++static inline float futrts_sinh32(float x) {+  return sinhf(x);+}++static inline float futrts_tanh32(float x) {+  return tanhf(x);+}++static inline float futrts_acosh32(float x) {+  return acoshf(x);+}++static inline float futrts_asinh32(float x) {+  return asinhf(x);+}++static inline float futrts_atanh32(float x) {+  return atanhf(x);+}++static inline float futrts_atan2_32(float x, float y) {+  return atan2f(x, y);+}++static inline float futrts_hypot32(float x, float y) {+  return hypotf(x, y);+}++static inline float futrts_gamma32(float x) {+  return tgammaf(x);+}++static inline float futrts_lgamma32(float x) {+  return lgammaf(x);+}++static inline float futrts_erf32(float x) {+  return erff(x);+}++static inline float futrts_erfc32(float x) {+  return erfcf(x);+}++static inline float fmod32(float x, float y) {+  return fmodf(x, y);+}++static inline float futrts_round32(float x) {+  return rintf(x);+}++static inline float futrts_floor32(float x) {+  return floorf(x);+}++static inline float futrts_ceil32(float x) {+  return ceilf(x);+}++static inline float futrts_lerp32(float v0, float v1, float t) {+  return v0 + (v1 - v0) * t;+}++static inline float futrts_mad32(float a, float b, float c) {+  return a * b + c;+}++static inline float futrts_fma32(float a, float b, float c) {+  return fmaf(a, b, c);+}+#endif++#if ISPC+static inline int32_t futrts_to_bits32(float x) {+  return intbits(x);+}++static inline float futrts_from_bits32(int32_t x) {+  return floatbits(x);+}+#else+static inline int32_t futrts_to_bits32(float x) {+  union {+    float f;+    int32_t t;+  } p;++  p.f = x;+  return p.t;+}++static inline float futrts_from_bits32(int32_t x) {+  union {+    int32_t f;+    float t;+  } p;++  p.f = x;+  return p.t;+}+#endif++static inline float fsignum32(float x) {+  return futrts_isnan32(x) ? x : (x > 0 ? 1 : 0) - (x < 0 ? 1 : 0);+}++#ifdef FUTHARK_F64_ENABLED++#if ISPC+static inline bool futrts_isinf64(float x) {+  return !isnan(x) && isnan(x - x);+}++static inline bool futrts_isfinite64(float x) {+  return !isnan(x) && !futrts_isinf64(x);+}++static inline double fdiv64(double x, double y) {+  return x / y;+}++static inline double fadd64(double x, double y) {+  return x + y;+}++static inline double fsub64(double x, double y) {+  return x - y;+}++static inline double fmul64(double x, double y) {+  return x * y;+}++static inline bool cmplt64(double x, double y) {+  return x < y;+}++static inline bool cmple64(double x, double y) {+  return x <= y;+}++static inline double sitofp_i8_f64(int8_t x) {+  return (double) x;+}++static inline double sitofp_i16_f64(int16_t x) {+  return (double) x;+}++static inline double sitofp_i32_f64(int32_t x) {+  return (double) x;+}++static inline double sitofp_i64_f64(int64_t x) {+  return (double) x;+}++static inline double uitofp_i8_f64(uint8_t x) {+  return (double) x;+}++static inline double uitofp_i16_f64(uint16_t x) {+  return (double) x;+}++static inline double uitofp_i32_f64(uint32_t x) {+  return (double) x;+}++static inline double uitofp_i64_f64(uint64_t x) {+  return (double) x;+}++static inline double fabs64(double x) {+  return abs(x);+}++static inline double fmax64(double x, double y) {+  return isnan(x) ? y : isnan(y) ? x : max(x, y);+}++static inline double fmin64(double x, double y) {+  return isnan(x) ? y : isnan(y) ? x : min(x, y);+}++static inline double fpow64(double a, double b) {+  float ret;+  foreach_active (i) {+      uniform float r = __stdlib_powf(extract(a, i), extract(b, i));+      ret = insert(ret, i, r);+  }+  return ret;+}++static inline double futrts_log64(double x) {+  return futrts_isfinite64(x) || (futrts_isinf64(x) && x < 0)? log(x) : x;+}++static inline double futrts_log2_64(double x) {+  return futrts_log64(x)/log(2.0d);+}++static inline double futrts_log10_64(double x) {+  return futrts_log64(x)/log(10.0d);+}++static inline double futrts_sqrt64(double x) {+  return sqrt(x);+}++extern "C" unmasked uniform double cbrt(uniform double);+static inline double futrts_cbrt64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = cbrtf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++static inline double futrts_exp64(double x) {+  return exp(x);+}++static inline double futrts_cos64(double x) {+  return cos(x);+}++static inline double futrts_sin64(double x) {+  return sin(x);+}++static inline double futrts_tan64(double x) {+  return tan(x);+}++static inline double futrts_acos64(double x) {+  return acos(x);+}++static inline double futrts_asin64(double x) {+  return asin(x);+}++static inline double futrts_atan64(double x) {+  return atan(x);+}++static inline double futrts_cosh64(double x) {+  return (exp(x)+exp(-x)) / 2.0d;+}++static inline double futrts_sinh64(double x) {+  return (exp(x)-exp(-x)) / 2.0d;+}++static inline double futrts_tanh64(double x) {+  return futrts_sinh64(x)/futrts_cosh64(x);+}++static inline double futrts_acosh64(double x) {+  double f = x+sqrt(x*x-1.0d);+  if(futrts_isfinite64(f)) return log(f);+  return f;+}++static inline double futrts_asinh64(double x) {+  double f = x+sqrt(x*x+1.0d);+  if(futrts_isfinite64(f)) return log(f);+  return f;+}++static inline double futrts_atanh64(double x) {+  double f = (1.0d+x)/(1.0d-x);+  if(futrts_isfinite64(f)) return log(f)/2.0d;+  return f;++}++static inline double futrts_atan2_64(double x, double y) {+  return atan2(x, y);+}++extern "C" unmasked uniform double hypot(uniform double x, uniform double y);+static inline double futrts_hypot64(double x, double y) {+  double res;+  foreach_active (i) {+    uniform double r = hypot(extract(x, i), extract(y, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform double tgamma(uniform double x);+static inline double futrts_gamma64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = tgamma(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform double lgamma(uniform double x);+static inline double futrts_lgamma64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = lgamma(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform double erf(uniform double x);+static inline double futrts_erf64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = erf(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform double erfc(uniform double x);+static inline double futrts_erfc64(double x) {+  double res;+  foreach_active (i) {+    uniform double r = erfc(extract(x, i));+    res = insert(res, i, r);+  }+  return res;+}++static inline double futrts_fma64(double a, double b, double c) {+  return a * b + c;+}++static inline double futrts_round64(double x) {+  return round(x);+}++static inline double futrts_ceil64(double x) {+  return ceil(x);+}++static inline double futrts_floor64(double x) {+  return floor(x);+}++static inline bool futrts_isnan64(double x) {+  return isnan(x);+}++static inline int8_t fptosi_f64_i8(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int8_t) x;+  }+}++static inline int16_t fptosi_f64_i16(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int16_t) x;+  }+}++static inline int32_t fptosi_f64_i32(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int32_t) x;+  }+}++static inline int64_t fptosi_f64_i64(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int64_t) x;+  }+}++static inline uint8_t fptoui_f64_i8(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint8_t) (int8_t) x;+  }+}++static inline uint16_t fptoui_f64_i16(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint16_t) (int16_t) x;+  }+}++static inline uint32_t fptoui_f64_i32(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint32_t) (int32_t) x;+  }+}++static inline uint64_t fptoui_f64_i64(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint64_t) (int64_t) x;+  }+}++static inline bool ftob_f64_bool(double x) {+  return x != 0.0;+}++static inline double btof_bool_f64(bool x) {+  return x ? 1.0 : 0.0;+}++static inline int64_t futrts_to_bits64(double x) {+  int64_t res;+  foreach_active (i) {+    uniform double tmp = extract(x, i);+    uniform int64_t r = *((uniform int64_t* uniform)&tmp);+    res = insert(res, i, r);+  }+  return res;+}++static inline double futrts_from_bits64(int64_t x) {+  double res;+  foreach_active (i) {+    uniform int64_t tmp = extract(x, i);+    uniform double r = *((uniform double* uniform)&tmp);+    res = insert(res, i, r);+  }+  return res;+}++static inline double fmod64(double x, double y) {+  return x - y * trunc(x/y);+}++static inline double fsignum64(double x) {+  return futrts_isnan64(x) ? x : (x > 0 ? 1.0d : 0.0d) - (x < 0 ? 1.0d : 0.0d);+}++static inline double futrts_lerp64(double v0, double v1, double t) {+  return v0 + (v1 - v0) * t;+}++static inline double futrts_mad64(double a, double b, double c) {+  return a * b + c;+}++static inline float fpconv_f32_f32(float x) {+  return (float) x;+}++static inline double fpconv_f32_f64(float x) {+  return (double) x;+}++static inline float fpconv_f64_f32(double x) {+  return (float) x;+}++static inline double fpconv_f64_f64(double x) {+  return (double) x;+}++#else++static inline double fdiv64(double x, double y) {+  return x / y;+}++static inline double fadd64(double x, double y) {+  return x + y;+}++static inline double fsub64(double x, double y) {+  return x - y;+}++static inline double fmul64(double x, double y) {+  return x * y;+}++static inline bool cmplt64(double x, double y) {+  return x < y;+}++static inline bool cmple64(double x, double y) {+  return x <= y;+}++static inline double sitofp_i8_f64(int8_t x) {+  return (double) x;+}++static inline double sitofp_i16_f64(int16_t x) {+  return (double) x;+}++static inline double sitofp_i32_f64(int32_t x) {+  return (double) x;+}++static inline double sitofp_i64_f64(int64_t x) {+  return (double) x;+}++static inline double uitofp_i8_f64(uint8_t x) {+  return (double) x;+}++static inline double uitofp_i16_f64(uint16_t x) {+  return (double) x;+}++static inline double uitofp_i32_f64(uint32_t x) {+  return (double) x;+}++static inline double uitofp_i64_f64(uint64_t x) {+  return (double) x;+}++static inline double fabs64(double x) {+  return fabs(x);+}++static inline double fmax64(double x, double y) {+  return fmax(x, y);+}++static inline double fmin64(double x, double y) {+  return fmin(x, y);+}++static inline double fpow64(double x, double y) {+  return pow(x, y);+}++static inline double futrts_log64(double x) {+  return log(x);+}++static inline double futrts_log2_64(double x) {+  return log2(x);+}++static inline double futrts_log10_64(double x) {+  return log10(x);+}++static inline double futrts_sqrt64(double x) {+  return sqrt(x);+}++static inline double futrts_cbrt64(double x) {+  return cbrt(x);+}++static inline double futrts_exp64(double x) {+  return exp(x);+}++static inline double futrts_cos64(double x) {+  return cos(x);+}++static inline double futrts_sin64(double x) {+  return sin(x);+}++static inline double futrts_tan64(double x) {+  return tan(x);+}++static inline double futrts_acos64(double x) {+  return acos(x);+}++static inline double futrts_asin64(double x) {+  return asin(x);+}++static inline double futrts_atan64(double x) {+  return atan(x);+}++static inline double futrts_cosh64(double x) {+  return cosh(x);+}++static inline double futrts_sinh64(double x) {+  return sinh(x);+}++static inline double futrts_tanh64(double x) {+  return tanh(x);+}++static inline double futrts_acosh64(double x) {+  return acosh(x);+}++static inline double futrts_asinh64(double x) {+  return asinh(x);+}++static inline double futrts_atanh64(double x) {+  return atanh(x);+}++static inline double futrts_atan2_64(double x, double y) {+  return atan2(x, y);+}++static inline double futrts_hypot64(double x, double y) {+  return hypot(x, y);+}++static inline double futrts_gamma64(double x) {+  return tgamma(x);+}++static inline double futrts_lgamma64(double x) {+  return lgamma(x);+}++static inline double futrts_erf64(double x) {+  return erf(x);+}++static inline double futrts_erfc64(double x) {+  return erfc(x);+}++static inline double futrts_fma64(double a, double b, double c) {+  return fma(a, b, c);+}++static inline double futrts_round64(double x) {+  return rint(x);+}++static inline double futrts_ceil64(double x) {+  return ceil(x);+}++static inline double futrts_floor64(double x) {+  return floor(x);+}++static inline bool futrts_isnan64(double x) {+  return isnan(x);+}++static inline bool futrts_isinf64(double x) {+  return isinf(x);+}++static inline int8_t fptosi_f64_i8(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int8_t) x;+  }+}++static inline int16_t fptosi_f64_i16(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int16_t) x;+  }+}++static inline int32_t fptosi_f64_i32(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int32_t) x;+  }+}++static inline int64_t fptosi_f64_i64(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (int64_t) x;+  }+}++static inline uint8_t fptoui_f64_i8(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint8_t) (int8_t) x;+  }+}++static inline uint16_t fptoui_f64_i16(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint16_t) (int16_t) x;+  }+}++static inline uint32_t fptoui_f64_i32(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint32_t) (int32_t) x;+  }+}++static inline uint64_t fptoui_f64_i64(double x) {+  if (futrts_isnan64(x) || futrts_isinf64(x)) {+    return 0;+  } else {+    return (uint64_t) (int64_t) x;+  }+}++static inline bool ftob_f64_bool(double x) {+  return x != 0;+}++static inline double btof_bool_f64(bool x) {+  return x ? 1 : 0;+}++static inline int64_t futrts_to_bits64(double x) {+  union {+    double f;+    int64_t t;+  } p;++  p.f = x;+  return p.t;+}++static inline double futrts_from_bits64(int64_t x) {+  union {+    int64_t f;+    double t;+  } p;++  p.f = x;+  return p.t;+}++static inline double fmod64(double x, double y) {+  return fmod(x, y);+}++static inline double fsignum64(double x) {+  return futrts_isnan64(x) ? x : (x > 0) - (x < 0);+}++static inline double futrts_lerp64(double v0, double v1, double t) {+#ifdef __OPENCL_VERSION__+  return mix(v0, v1, t);+#else+  return v0 + (v1 - v0) * t;+#endif+}++static inline double futrts_mad64(double a, double b, double c) {+#ifdef __OPENCL_VERSION__+  return mad(a, b, c);+#else+  return a * b + c;+#endif+}++static inline float fpconv_f32_f32(float x) {+  return (float) x;+}++static inline double fpconv_f32_f64(float x) {+  return (double) x;+}++static inline float fpconv_f64_f32(double x) {+  return (float) x;+}++static inline double fpconv_f64_f64(double x) {+  return (double) x;+}++#endif  #endif 

@@ -9,7 +9,7 @@ // under emulation, so the compiler will have to be careful when // generating reads or writes. -#if !defined(cl_khr_fp16) && !(defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 600)+#if !defined(cl_khr_fp16) && !(defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 600) && !(defined(ISPC)) #define EMULATE_F16 #endif @@ -23,6 +23,9 @@ // compiler will have to be real careful! typedef float f16; +#elif ISPC+typedef float16 f16;+ #else  #ifdef __CUDA_ARCH__@@ -129,8 +132,12 @@ }  #ifndef EMULATE_F16+static inline bool futrts_isnan16(f16 x) {+  return isnan((float)x);+}  #ifdef __OPENCL_VERSION__+ static inline f16 fabs16(f16 x) {   return fabs(x); }@@ -147,6 +154,22 @@   return pow(x, y); } +#elif ISPC+static inline f16 fabs16(f16 x) {+  return abs(x);+}++static inline f16 fmax16(f16 x, f16 y) {+  return futrts_isnan16(x) ? y : futrts_isnan16(y) ? x : max(x, y);+}++static inline f16 fmin16(f16 x, f16 y) {+  return futrts_isnan16(x) ? y : futrts_isnan16(y) ? x : min(x, y);+}++static inline f16 fpow16(f16 x, f16 y) {+  return pow(x, y);+} #else // Assuming CUDA.  static inline f16 fabs16(f16 x) {@@ -166,13 +189,20 @@ } #endif -static inline bool futrts_isnan16(f16 x) {-  return isnan((float)x);+#if ISPC+static inline bool futrts_isinf16(float x) {+  return !futrts_isnan16(x) && futrts_isnan16(x - x); }+static inline bool futrts_isfinite16(float x) {+  return !futrts_isnan16(x) && !futrts_isinf16(x);+} +#else+ static inline bool futrts_isinf16(f16 x) {   return isinf((float)x); }+#endif  #ifdef __OPENCL_VERSION__ static inline f16 futrts_log16(f16 x) {@@ -298,7 +328,153 @@ static inline f16 futrts_fma16(f16 a, f16 b, f16 c) {   return fma(a, b, c); }+#elif ISPC +static inline f16 futrts_log16(f16 x) {+  return futrts_isfinite16(x) || (futrts_isinf16(x) && x < 0) ? log(x) : x;+}++static inline f16 futrts_log2_16(f16 x) {+  return futrts_log16(x) / log(2.0f16);+}++static inline f16 futrts_log10_16(f16 x) {+  return futrts_log16(x) / log(10.0f16);+}++static inline f16 futrts_sqrt16(f16 x) {+  return (float16)sqrt((float)x);+}++static inline f16 futrts_exp16(f16 x) {+  return exp(x);+}++static inline f16 futrts_cos16(f16 x) {+  return (float16)cos((float)x);+}++static inline f16 futrts_sin16(f16 x) {+  return (float16)sin((float)x);+}++static inline f16 futrts_tan16(f16 x) {+  return (float16)tan((float)x);+}++static inline f16 futrts_acos16(f16 x) {+  return (float16)acos((float)x);+}++static inline f16 futrts_asin16(f16 x) {+  return (float16)asin((float)x);+}++static inline f16 futrts_atan16(f16 x) {+  return (float16)atan((float)x);+}++static inline f16 futrts_cosh16(f16 x) {+  return (exp(x)+exp(-x)) / 2.0f16;+}++static inline f16 futrts_sinh16(f16 x) {+  return (exp(x)-exp(-x)) / 2.0f16;+}++static inline f16 futrts_tanh16(f16 x) {+  return futrts_sinh16(x)/futrts_cosh16(x);+}++static inline f16 futrts_acosh16(f16 x) {+  float16 f = x+(float16)sqrt((float)(x*x-1));+  if(futrts_isfinite16(f)) return log(f);+  return f;+}++static inline f16 futrts_asinh16(f16 x) {+  float16 f = x+(float16)sqrt((float)(x*x+1));+  if(futrts_isfinite16(f)) return log(f);+  return f;+}++static inline f16 futrts_atanh16(f16 x) {+  float16 f = (1+x)/(1-x);+  if(futrts_isfinite16(f)) return log(f)/2.0f16;+  return f;+}++static inline f16 futrts_atan2_16(f16 x, f16 y) {+  return (float16)atan2((float)x, (float)y);+}++static inline f16 futrts_hypot16(f16 x, f16 y) {+  return (float16)futrts_hypot32((float)x, (float)y);+}++extern "C" unmasked uniform float tgammaf(uniform float x);+static inline f16 futrts_gamma16(f16 x) {+  f16 res;+  foreach_active (i) {+    uniform f16 r = (f16)tgammaf(extract((float)x, i));+    res = insert(res, i, r);+  }+  return res;+}++extern "C" unmasked uniform float lgammaf(uniform float x);+static inline f16 futrts_lgamma16(f16 x) {+  f16 res;+  foreach_active (i) {+    uniform f16 r = (f16)lgammaf(extract((float)x, i));+    res = insert(res, i, r);+  }+  return res;+}++static inline f16 futrts_cbrt16(f16 x) {+  f16 res = (f16)futrts_cbrt32((float)x);+  return res;+}++static inline f16 futrts_erf16(f16 x) {+  f16 res = (f16)futrts_erf32((float)x);+  return res;+}++static inline f16 futrts_erfc16(f16 x) {+  f16 res = (f16)futrts_erfc32((float)x);+  return res;+}++static inline f16 fmod16(f16 x, f16 y) {+  return x - y * (float16)trunc((float) (x/y));+}++static inline f16 futrts_round16(f16 x) {+  return (float16)round((float)x);+}++static inline f16 futrts_floor16(f16 x) {+  return (float16)floor((float)x);+}++static inline f16 futrts_ceil16(f16 x) {+  return (float16)ceil((float)x);+}++static inline f16 futrts_lerp16(f16 v0, f16 v1, f16 t) {+  return v0 + (v1 - v0) * t;+}++static inline f16 futrts_mad16(f16 a, f16 b, f16 c) {+  return a * b + c;+}++static inline f16 futrts_fma16(f16 a, f16 b, f16 c) {+  return a * b + c;+}+ #else // Assume CUDA.  static inline f16 futrts_log16(f16 x) {@@ -436,6 +612,17 @@ static inline f16 futrts_from_bits16(int16_t x) {   return __ushort_as_half(x); }+#elif ISPC++static inline int16_t futrts_to_bits16(f16 x) {+  varying int16_t y = *((varying int16_t * uniform)&x);+  return y;+}++static inline f16 futrts_from_bits16(int16_t x) {+  varying f16 y = *((varying f16 * uniform)&x);+  return y;+} #else static inline int16_t futrts_to_bits16(f16 x) {   union {@@ -636,7 +823,7 @@ }  static inline f16 fsignum16(f16 x) {-  return futrts_isnan16(x) ? x : (x > 0) - (x < 0);+  return futrts_isnan16(x) ? x : (x > 0 ? 1 : 0) - (x < 0 ? 1 : 0); }  #endif@@ -652,7 +839,7 @@ }  static inline f16 fpconv_f32_f16(float x) {-  return x;+  return (f16) x; }  #ifdef FUTHARK_F64_ENABLED@@ -661,10 +848,15 @@   return (double) x; } +#if ISPC static inline f16 fpconv_f64_f16(double x) {+  return (f16) ((float)x);+}+#else+static inline f16 fpconv_f64_f16(double x) {   return (f16) x; }-+#endif #endif  

@@ -495,7 +495,9 @@ struct scheduler {   struct worker *workers;   int num_threads;+  int minimum_chunk_size; +   // If there is work to steal => active_work > 0   volatile int active_work; @@ -534,12 +536,11 @@   return i; } --static inline int64_t compute_chunk_size(double kappa, struct subtask* subtask)+static inline int64_t compute_chunk_size(int64_t minimum_chunk_size, double kappa, struct subtask* subtask) {   double C = (double)*subtask->task_time / (double)*subtask->task_iter;   if (C == 0.0F) C += DBL_EPSILON;-  return smax64((int64_t)(kappa / C), 1);+  return smax64((int64_t)(kappa / C), minimum_chunk_size); }  /* Takes a chunk from subtask and enqueues the remaining iterations onto the worker's queue */@@ -549,7 +550,9 @@   if (subtask->chunkable) {     // Do we have information from previous runs avaliable     if (*subtask->task_iter > 0) {-      subtask->chunk_size = compute_chunk_size(worker->scheduler->kappa, subtask);+      subtask->chunk_size = compute_chunk_size(worker->scheduler->minimum_chunk_size,+                                               worker->scheduler->kappa,+                                               subtask);       assert(subtask->chunk_size > 0);     }     int64_t remaining_iter = subtask->end - subtask->start;@@ -790,7 +793,7 @@   enum scheduling sched = info.sched;   /* If each subtasks should be processed in chunks */   int chunkable = sched == STATIC ? 0 : 1;-  int64_t chunk_size = 1; // The initial chunk size when no info is avaliable+  int64_t chunk_size = scheduler->minimum_chunk_size; // The initial chunk size when no info is avaliable     if (info.wake_up_threads || sched == DYNAMIC)@@ -1076,6 +1079,13 @@ static int scheduler_init(struct scheduler *scheduler,                           int num_workers,                           double kappa) {+#ifdef FUTHARK_BACKEND_ispc+  int64_t get_gang_size();+  scheduler->minimum_chunk_size = get_gang_size();+#else+  scheduler->minimum_chunk_size = 1;+#endif+   assert(num_workers > 0);    scheduler->kappa = kappa;

@@ -14,13 +14,28 @@ typedef int (*restore_fn)(const void*, FILE *, struct futhark_context*, void*); typedef void (*store_fn)(const void*, FILE *, struct futhark_context*, void*); typedef int (*free_fn)(const void*, struct futhark_context*, void*);+typedef int (*project_fn)(struct futhark_context*, void*, const void*);+typedef int (*new_fn)(struct futhark_context*, void**, const void*[]); +struct field {+  const char *name;+  const struct type *type;+  project_fn project;+};++struct record {+  int num_fields;+  const struct field* fields;+  new_fn new;+};+ struct type {   const char *name;   restore_fn restore;   store_fn store;   free_fn free;   const void *aux;+  const struct record *record; };  int free_scalar(const void *aux, struct futhark_context *ctx, void *p) {@@ -31,27 +46,27 @@   return 0; } -#define DEF_SCALAR_TYPE(T)                                              \-  int restore_##T(const void *aux, FILE *f,                             \-                  struct futhark_context *ctx, void *p) {               \-    (void)aux;                                                          \-    (void)ctx;                                                          \-    return read_scalar(f, &T##_info, p);                                \-  }                                                                     \-                                                                        \-  void store_##T(const void *aux, FILE *f,                              \-                 struct futhark_context *ctx, void *p) {                \-    (void)aux;                                                          \-    (void)ctx;                                                          \-    write_scalar(f, 1, &T##_info, p);                                   \-  }                                                                     \-                                                                        \-  struct type type_##T =                                                \-    { .name = #T,                                                       \-      .restore = restore_##T,                                           \-      .store = store_##T,                                               \-      .free = free_scalar                                               \-    }                                                                   \+#define DEF_SCALAR_TYPE(T)                                      \+  int restore_##T(const void *aux, FILE *f,                     \+                  struct futhark_context *ctx, void *p) {       \+    (void)aux;                                                  \+    (void)ctx;                                                  \+    return read_scalar(f, &T##_info, p);                        \+  }                                                             \+                                                                \+  void store_##T(const void *aux, FILE *f,                      \+                 struct futhark_context *ctx, void *p) {        \+    (void)aux;                                                  \+    (void)ctx;                                                  \+    write_scalar(f, 1, &T##_info, p);                           \+  }                                                             \+                                                                \+  struct type type_##T =                                        \+    { .name = #T,                                               \+      .restore = restore_##T,                                   \+      .store = store_##T,                                       \+      .free = free_scalar                                       \+    }                                                           \  DEF_SCALAR_TYPE(i8); DEF_SCALAR_TYPE(i16);@@ -67,7 +82,7 @@ DEF_SCALAR_TYPE(bool);  struct value {-  struct type *type;+  const struct type *type;   union {     void *v_ptr;     int8_t  v_i8;@@ -139,9 +154,9 @@ struct entry_point {   const char *name;   entry_point_fn f;-  struct type **out_types;+  const struct type **out_types;   bool *out_unique;-  struct type **in_types;+  const struct type **in_types;   bool *in_unique; }; @@ -165,7 +180,7 @@   // Last entry point identified by NULL name.   struct entry_point *entry_points;   // Last type identified by NULL name.-  struct type **types;+  const struct type **types; };  struct server_state {@@ -190,7 +205,7 @@  struct variable* create_variable(struct server_state *s,                                  const char *name,-                                 struct type *type) {+                                 const struct type *type) {   int found = -1;   for (int i = 0; i < s->variables_capacity; i++) {     if (found == -1 && s->variables[i].name == NULL) {@@ -240,7 +255,7 @@   return args[i]; } -struct type* get_type(struct server_state *s, const char *name) {+const struct type* get_type(struct server_state *s, const char *name) {   for (int i = 0; s->prog.types[i]; i++) {     if (strcmp(s->prog.types[i]->name, name) == 0) {       return s->prog.types[i];@@ -364,7 +379,7 @@       const char *vname = get_arg(args, i);       const char *type = get_arg(args, i+1); -      struct type *t = get_type(s, type);+      const struct type *t = get_type(s, type);       struct variable *v = create_variable(s, vname, t);        if (v == NULL) {@@ -414,7 +429,7 @@         return;       } -      struct type *t = v->value.type;+      const struct type *t = v->value.type;       t->store(t->aux, f, s->ctx, value_ptr(&v->value));     }     fclose(f);@@ -432,7 +447,7 @@       return;     } -    struct type *t = v->value.type;+    const struct type *t = v->value.type;      int err = t->free(t->aux, s->ctx, value_ptr(&v->value));     error_check(s, err);@@ -544,6 +559,138 @@   } } +void cmd_fields(struct server_state *s, const char *args[]) {+  const char *type = get_arg(args, 0);+  const struct type *t = get_type(s, type);+  const struct record *r = t->record;++  if (r == NULL) {+    failure();+    printf("Not a record type\n");+    return;+  }++  for (int i = 0; i < r->num_fields; i++) {+    const struct field f = r->fields[i];+    printf("%s %s\n", f.name, f.type->name);+  }+}++void cmd_project(struct server_state *s, const char *args[]) {+  const char *to_name = get_arg(args, 0);+  const char *from_name = get_arg(args, 1);+  const char *field_name = get_arg(args, 2);++  struct variable *from = get_variable(s, from_name);++  if (from == NULL) {+    failure();+    printf("Unknown variable: %s\n", from_name);+    return;+  }++  const struct type *from_type = from->value.type;+  const struct record *r = from_type->record;++  if (r == NULL) {+    failure();+    printf("Not a record type\n");+    return;+  }++  const struct field *field = NULL;+  for (int i = 0; i < r->num_fields; i++) {+    if (strcmp(r->fields[i].name, field_name) == 0) {+      field = &r->fields[i];+      break;+    }+  }++  if (field == NULL) {+    failure();+    printf("No such field\n");+  }++  struct variable *to = create_variable(s, to_name, field->type);++  if (to == NULL) {+    failure();+    printf("Variable already exists: %s\n", to_name);+    return;+  }++  field->project(s->ctx, value_ptr(&to->value), from->value.value.v_ptr);+}++void cmd_new(struct server_state *s, const char *args[]) {+  const char *to_name = get_arg(args, 0);+  const char *type_name = get_arg(args, 1);+  const struct type *type = get_type(s, type_name);+  struct variable *to = create_variable(s, to_name, type);++  if (to == NULL) {+    failure();+    printf("Variable already exists: %s\n", to_name);+    return;+  }++  const struct record* r = type->record;++  if (r == NULL) {+    failure();+    printf("Not a record type\n");+    return;+  }++  int num_args = 0;+  for (int i = 2; arg_exists(args, i); i++) {+    num_args++;+  }++  if (num_args != r->num_fields) {+    failure();+    printf("%d fields expected byt %d values provided.\n", num_args, r->num_fields);+    return;+  }++  const void** value_ptrs = alloca(num_args * sizeof(void*));++  for (int i = 0; i < num_args; i++) {+    struct variable* v = get_variable(s, args[2+i]);++    if (v == NULL) {+      failure();+      printf("Unknown variable: %s\n", args[2+i]);+      return;+    }++    if (strcmp(v->value.type->name, r->fields[i].type->name) != 0) {+      failure();+      printf("Field %s mismatch: expected type %s, got %s\n",+             r->fields[i].name, r->fields[i].type->name, v->value.type->name);+      return;+    }++    value_ptrs[i] = value_ptr(&v->value);+  }++  r->new(s->ctx, value_ptr(&to->value), value_ptrs);+}++void cmd_entry_points(struct server_state *s, const char *args[]) {+  (void)args;+  for (int i = 0; s->prog.entry_points[i].name; i++) {+    puts(s->prog.entry_points[i].name);+  }+}++void cmd_types(struct server_state *s, const char *args[]) {+  (void)args;+  for (int i = 0; s->prog.types[i] != NULL; i++) {+    puts(s->prog.types[i]->name);+  }+}+ char *next_word(char **line) {   char *p = *line; @@ -630,6 +777,16 @@     cmd_report(s, tokens+1);   } else if (strcmp(command, "set_tuning_param") == 0) {     cmd_set_tuning_param(s, tokens+1);+  } else if (strcmp(command, "fields") == 0) {+    cmd_fields(s, tokens+1);+  } else if (strcmp(command, "new") == 0) {+    cmd_new(s, tokens+1);+  } else if (strcmp(command, "project") == 0) {+    cmd_project(s, tokens+1);+  } else if (strcmp(command, "entry_points") == 0) {+    cmd_entry_points(s, tokens+1);+  } else if (strcmp(command, "types") == 0) {+    cmd_types(s, tokens+1);   } else {     futhark_panic(1, "Unknown command: %s\n", command);   }@@ -695,6 +852,7 @@   if (arr == NULL) {     return 1;   }+  assert(futhark_context_sync(ctx) == 0);    *(void**)p = arr;   free(data);@@ -717,7 +875,7 @@ }  int free_array(const struct array_aux *aux,-                struct futhark_context *ctx, void *p) {+               struct futhark_context *ctx, void *p) {   void *arr = *(void**)p;   return aux->free(ctx, arr); }@@ -769,7 +927,7 @@ }  int free_opaque(const struct opaque_aux *aux,-                 struct futhark_context *ctx, void *p) {+                struct futhark_context *ctx, void *p) {   void *obj = *(void**)p;   return aux->free(ctx, obj); }