diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,3 @@
+# 0.1.0.0
+
+* Initial release.
diff --git a/LICENSE b/LICENSE
new file mode 100644
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,661 @@
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+.
diff --git a/bz3.cabal b/bz3.cabal
new file mode 100644
--- /dev/null
+++ b/bz3.cabal
@@ -0,0 +1,59 @@
+cabal-version: 3.4
+name: bz3
+version: 0.1.0.0
+synopsis: High-level bindings to bz3
+description:
+ Streaming compression/decompression in bz3 format via lazy bytestrings
+license: AGPL-3.0-or-later
+license-file: LICENSE
+author: Vanessa McHale
+maintainer: vamchale@gmail.com
+category: Codec
+build-type: Simple
+extra-doc-files: CHANGELOG.md
+extra-source-files:
+ include/common.h
+ include/libsais.h
+
+source-repository head
+ type: darcs
+ location: https://hub.darcs.net/vmchale/bz3
+
+common warnings
+ ghc-options:
+ -Wall -fno-warn-missing-signatures -Wno-x-partial
+ -Wincomplete-uni-patterns -Wincomplete-record-updates
+ -Wredundant-constraints -Wmissing-export-lists -Wcpp-undef
+ -Wunused-packages -Wno-operator-whitespace-ext-conflict
+
+library
+ import: warnings
+ exposed-modules: Codec.Bz3
+ other-modules: Codec.Bz3.Foreign
+ Codec.Bz3.Binary
+ other-extensions: OverloadedStrings
+ cc-options: -DVERSION="1.5.1"
+ c-sources: c/libbz3.c
+ include-dirs: include
+ install-includes: include/libbz3.h
+ build-depends: base >=4.16.0.0 && <5
+ , bytestring >=0.11.0.0
+ , binary >=0.6.4.0
+ build-tool-depends: c2hs:c2hs
+ hs-source-dirs: src
+ default-language: Haskell2010
+
+test-suite bz3-test
+ import: warnings
+ default-language: Haskell2010
+ type: exitcode-stdio-1.0
+ hs-source-dirs: test
+ main-is: Main.hs
+ ghc-options: -threaded -rtsopts "-with-rtsopts=-N -k1k"
+ build-depends:
+ base,
+ bz3,
+ bytestring,
+ directory >=1.2.5.0,
+ tasty,
+ tasty-hunit
diff --git a/c/libbz3.c b/c/libbz3.c
new file mode 100644
--- /dev/null
+++ b/c/libbz3.c
@@ -0,0 +1,1053 @@
+
+/*
+ * BZip3 - A spiritual successor to BZip2.
+ * Copyright (C) 2022-2024 Kamila Szewczyk
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License along with
+ * this program. If not, see .
+ */
+
+#include "libbz3.h"
+#include
+#include
+#include "libsais.h"
+
+#if defined(__GNUC__) || defined(__clang__)
+ #define LIKELY(x) __builtin_expect(!!(x), 1)
+ #define UNLIKELY(x) __builtin_expect(!!(x), 0)
+#else
+ #define LIKELY(x) (x)
+ #define UNLIKELY(x) (x)
+#endif
+
+/* CRC32 implementation. Since CRC32 generally takes less than 1% of the runtime on real-world data (e.g. the
+ Silesia corpus), I decided against using hardware CRC32. This implementation is simple, fast, fool-proof and
+ good enough to be used with bzip3. */
+
+static const u32 crc32Table[256] = {
+ 0x00000000L, 0xF26B8303L, 0xE13B70F7L, 0x1350F3F4L, 0xC79A971FL, 0x35F1141CL, 0x26A1E7E8L, 0xD4CA64EBL, 0x8AD958CFL,
+ 0x78B2DBCCL, 0x6BE22838L, 0x9989AB3BL, 0x4D43CFD0L, 0xBF284CD3L, 0xAC78BF27L, 0x5E133C24L, 0x105EC76FL, 0xE235446CL,
+ 0xF165B798L, 0x030E349BL, 0xD7C45070L, 0x25AFD373L, 0x36FF2087L, 0xC494A384L, 0x9A879FA0L, 0x68EC1CA3L, 0x7BBCEF57L,
+ 0x89D76C54L, 0x5D1D08BFL, 0xAF768BBCL, 0xBC267848L, 0x4E4DFB4BL, 0x20BD8EDEL, 0xD2D60DDDL, 0xC186FE29L, 0x33ED7D2AL,
+ 0xE72719C1L, 0x154C9AC2L, 0x061C6936L, 0xF477EA35L, 0xAA64D611L, 0x580F5512L, 0x4B5FA6E6L, 0xB93425E5L, 0x6DFE410EL,
+ 0x9F95C20DL, 0x8CC531F9L, 0x7EAEB2FAL, 0x30E349B1L, 0xC288CAB2L, 0xD1D83946L, 0x23B3BA45L, 0xF779DEAEL, 0x05125DADL,
+ 0x1642AE59L, 0xE4292D5AL, 0xBA3A117EL, 0x4851927DL, 0x5B016189L, 0xA96AE28AL, 0x7DA08661L, 0x8FCB0562L, 0x9C9BF696L,
+ 0x6EF07595L, 0x417B1DBCL, 0xB3109EBFL, 0xA0406D4BL, 0x522BEE48L, 0x86E18AA3L, 0x748A09A0L, 0x67DAFA54L, 0x95B17957L,
+ 0xCBA24573L, 0x39C9C670L, 0x2A993584L, 0xD8F2B687L, 0x0C38D26CL, 0xFE53516FL, 0xED03A29BL, 0x1F682198L, 0x5125DAD3L,
+ 0xA34E59D0L, 0xB01EAA24L, 0x42752927L, 0x96BF4DCCL, 0x64D4CECFL, 0x77843D3BL, 0x85EFBE38L, 0xDBFC821CL, 0x2997011FL,
+ 0x3AC7F2EBL, 0xC8AC71E8L, 0x1C661503L, 0xEE0D9600L, 0xFD5D65F4L, 0x0F36E6F7L, 0x61C69362L, 0x93AD1061L, 0x80FDE395L,
+ 0x72966096L, 0xA65C047DL, 0x5437877EL, 0x4767748AL, 0xB50CF789L, 0xEB1FCBADL, 0x197448AEL, 0x0A24BB5AL, 0xF84F3859L,
+ 0x2C855CB2L, 0xDEEEDFB1L, 0xCDBE2C45L, 0x3FD5AF46L, 0x7198540DL, 0x83F3D70EL, 0x90A324FAL, 0x62C8A7F9L, 0xB602C312L,
+ 0x44694011L, 0x5739B3E5L, 0xA55230E6L, 0xFB410CC2L, 0x092A8FC1L, 0x1A7A7C35L, 0xE811FF36L, 0x3CDB9BDDL, 0xCEB018DEL,
+ 0xDDE0EB2AL, 0x2F8B6829L, 0x82F63B78L, 0x709DB87BL, 0x63CD4B8FL, 0x91A6C88CL, 0x456CAC67L, 0xB7072F64L, 0xA457DC90L,
+ 0x563C5F93L, 0x082F63B7L, 0xFA44E0B4L, 0xE9141340L, 0x1B7F9043L, 0xCFB5F4A8L, 0x3DDE77ABL, 0x2E8E845FL, 0xDCE5075CL,
+ 0x92A8FC17L, 0x60C37F14L, 0x73938CE0L, 0x81F80FE3L, 0x55326B08L, 0xA759E80BL, 0xB4091BFFL, 0x466298FCL, 0x1871A4D8L,
+ 0xEA1A27DBL, 0xF94AD42FL, 0x0B21572CL, 0xDFEB33C7L, 0x2D80B0C4L, 0x3ED04330L, 0xCCBBC033L, 0xA24BB5A6L, 0x502036A5L,
+ 0x4370C551L, 0xB11B4652L, 0x65D122B9L, 0x97BAA1BAL, 0x84EA524EL, 0x7681D14DL, 0x2892ED69L, 0xDAF96E6AL, 0xC9A99D9EL,
+ 0x3BC21E9DL, 0xEF087A76L, 0x1D63F975L, 0x0E330A81L, 0xFC588982L, 0xB21572C9L, 0x407EF1CAL, 0x532E023EL, 0xA145813DL,
+ 0x758FE5D6L, 0x87E466D5L, 0x94B49521L, 0x66DF1622L, 0x38CC2A06L, 0xCAA7A905L, 0xD9F75AF1L, 0x2B9CD9F2L, 0xFF56BD19L,
+ 0x0D3D3E1AL, 0x1E6DCDEEL, 0xEC064EEDL, 0xC38D26C4L, 0x31E6A5C7L, 0x22B65633L, 0xD0DDD530L, 0x0417B1DBL, 0xF67C32D8L,
+ 0xE52CC12CL, 0x1747422FL, 0x49547E0BL, 0xBB3FFD08L, 0xA86F0EFCL, 0x5A048DFFL, 0x8ECEE914L, 0x7CA56A17L, 0x6FF599E3L,
+ 0x9D9E1AE0L, 0xD3D3E1ABL, 0x21B862A8L, 0x32E8915CL, 0xC083125FL, 0x144976B4L, 0xE622F5B7L, 0xF5720643L, 0x07198540L,
+ 0x590AB964L, 0xAB613A67L, 0xB831C993L, 0x4A5A4A90L, 0x9E902E7BL, 0x6CFBAD78L, 0x7FAB5E8CL, 0x8DC0DD8FL, 0xE330A81AL,
+ 0x115B2B19L, 0x020BD8EDL, 0xF0605BEEL, 0x24AA3F05L, 0xD6C1BC06L, 0xC5914FF2L, 0x37FACCF1L, 0x69E9F0D5L, 0x9B8273D6L,
+ 0x88D28022L, 0x7AB90321L, 0xAE7367CAL, 0x5C18E4C9L, 0x4F48173DL, 0xBD23943EL, 0xF36E6F75L, 0x0105EC76L, 0x12551F82L,
+ 0xE03E9C81L, 0x34F4F86AL, 0xC69F7B69L, 0xD5CF889DL, 0x27A40B9EL, 0x79B737BAL, 0x8BDCB4B9L, 0x988C474DL, 0x6AE7C44EL,
+ 0xBE2DA0A5L, 0x4C4623A6L, 0x5F16D052L, 0xAD7D5351L
+};
+
+static u32 crc32sum(u32 crc, u8 * RESTRICT buf, size_t size) {
+ while (size--) crc = crc32Table[((u8)crc ^ *(buf++)) & 0xff] ^ (crc >> 8);
+ return crc;
+}
+
+/* LZP code. These constants were manually tuned to give the best compression ratio while using relatively
+ little resources. The LZP dictionary is only around 1MiB in size and the minimum match length was chosen
+ so that LZP would not interfere too much with the Burrows-Wheeler transform and the arithmetic coder, and
+ just collapse long redundant data instead (for a major speed-up at a low compression ratio cost - in fact,
+ LZP preprocessing often improves compression in some cases). */
+
+/* A heavily modified version of libbsc's LZP predictor w/ unaligned accesses follows. This one has single thread
+ performance and provides better compression ratio. It is also mostly UB-free and less brittle during
+ AFL fuzzing. */
+
+#define LZP_DICTIONARY 18
+#define LZP_MIN_MATCH 40
+
+#define MATCH 0xf2
+
+static u32 lzp_upcast(const u8 * ptr) {
+ // val = *(u32 *)ptr; - written this way to avoid UB
+ u32 val;
+ memcpy(&val, ptr, sizeof(val));
+ return val;
+}
+
+/**
+ * @brief Check if the buffer size is sufficient for decoding a bz3 block
+ *
+ * Data passed to the last step can be one of the following:
+ * - original data
+ * - original data + LZP
+ * - original data + RLE
+ * - original data + RLE + LZP
+ *
+ * We must ensure `buffer_size` is large enough to store the data at every step
+ * when walking backwards. The required size may be stored in either `lzp_size`,
+ * `rle_size` OR `orig_size`.
+ *
+ * @param buffer_size Size of the output buffer
+ * @param lzp_size Size after LZP decompression (-1 if LZP not used)
+ * @param rle_size Size after RLE decompression (-1 if RLE not used)
+ * @return 1 if buffer size is sufficient, 0 otherwise
+ */
+static int bz3_check_buffer_size(size_t buffer_size, s32 lzp_size, s32 rle_size, s32 orig_size) {
+ // Handle -1 cases to avoid implicit conversion issues
+ size_t effective_lzp_size = lzp_size < 0 ? 0 : (size_t)lzp_size;
+ size_t effective_rle_size = rle_size < 0 ? 0 : (size_t)rle_size;
+ size_t effective_orig_size = orig_size < 0 ? 0 : (size_t)orig_size;
+
+ // Check if buffer can hold intermediate results
+ return (effective_lzp_size <= buffer_size) && (effective_rle_size <= buffer_size) && (effective_orig_size <= buffer_size);
+}
+
+static s32 lzp_encode_block(const u8 * RESTRICT in, const u8 * in_end, u8 * RESTRICT out, u8 * out_end,
+ s32 * RESTRICT lut) {
+ const u8 * ins = in;
+ const u8 * outs = out;
+ const u8 * out_eob = out_end - 8;
+ const u8 * heur = in;
+
+ u32 ctx;
+
+ for (s32 i = 0; i < 4; ++i) *out++ = *in++;
+
+ ctx = ((u32)in[-1]) | (((u32)in[-2]) << 8) | (((u32)in[-3]) << 16) | (((u32)in[-4]) << 24);
+
+ while (in < in_end - LZP_MIN_MATCH - 32 && out < out_eob) {
+ u32 idx = (ctx >> 15 ^ ctx ^ ctx >> 3) & ((s32)(1 << LZP_DICTIONARY) - 1);
+ s32 val = lut[idx];
+ lut[idx] = in - ins;
+ if (val > 0) {
+ const u8 * RESTRICT ref = ins + val;
+ if (memcmp(in + LZP_MIN_MATCH - 4, ref + LZP_MIN_MATCH - 4, sizeof(u32)) == 0 &&
+ memcmp(in, ref, sizeof(u32)) == 0) {
+ if (heur > in && lzp_upcast(heur) != lzp_upcast(ref + (heur - in))) goto not_found;
+
+ s32 len = 4;
+ for (; in + len < in_end - LZP_MIN_MATCH - 32; len += sizeof(u32)) {
+ if (lzp_upcast(in + len) != lzp_upcast(ref + len)) break;
+ }
+
+ if (len < LZP_MIN_MATCH) {
+ if (heur < in + len) heur = in + len;
+ goto not_found;
+ }
+
+ len += in[len] == ref[len];
+ len += in[len] == ref[len];
+ len += in[len] == ref[len];
+
+ in += len;
+ ctx = ((u32)in[-1]) | (((u32)in[-2]) << 8) | (((u32)in[-3]) << 16) | (((u32)in[-4]) << 24);
+
+ *out++ = MATCH;
+
+ len -= LZP_MIN_MATCH;
+ while (len >= 254) {
+ len -= 254;
+ *out++ = 254;
+ if (out >= out_eob) break;
+ }
+
+ *out++ = len;
+ } else {
+ not_found:;
+ u8 next = *out++ = *in++;
+ ctx = ctx << 8 | next;
+ if (next == MATCH) *out++ = 255;
+ }
+ } else {
+ ctx = (ctx << 8) | (*out++ = *in++);
+ }
+ }
+
+ ctx = ((u32)in[-1]) | (((u32)in[-2]) << 8) | (((u32)in[-3]) << 16) | (((u32)in[-4]) << 24);
+
+ while (in < in_end && out < out_eob) {
+ u32 idx = (ctx >> 15 ^ ctx ^ ctx >> 3) & ((s32)(1 << LZP_DICTIONARY) - 1);
+ s32 val = lut[idx];
+ lut[idx] = (s32)(in - ins);
+
+ u8 next = *out++ = *in++;
+ ctx = ctx << 8 | next;
+ if (next == MATCH && val > 0) *out++ = 255;
+ }
+
+ return out >= out_eob ? -1 : (s32)(out - outs);
+}
+
+static s32 lzp_decode_block(const u8 * RESTRICT in, const u8 * in_end, s32 * RESTRICT lut, u8 * RESTRICT out,
+ const u8 * out_end) {
+ const u8 * outs = out;
+
+ for (s32 i = 0; i < 4; ++i) *out++ = *in++;
+
+ u32 ctx = ((u32)out[-1]) | (((u32)out[-2]) << 8) | (((u32)out[-3]) << 16) | (((u32)out[-4]) << 24);
+
+ while (in < in_end && out < out_end) {
+ u32 idx = (ctx >> 15 ^ ctx ^ ctx >> 3) & ((s32)(1 << LZP_DICTIONARY) - 1);
+ s32 val = lut[idx]; // SAFETY: guaranteed to be in-bounds by & mask.
+ lut[idx] = (s32)(out - outs);
+ if (*in == MATCH && val > 0) {
+ in++;
+ // SAFETY: 'in' is advanced here, but it may have been at last index in the case of untrusted bad data.
+ if (UNLIKELY(in == in_end)) return -1;
+ if (*in != 255) {
+ s32 len = LZP_MIN_MATCH;
+ while (1) {
+ if (UNLIKELY(in == in_end)) return -1;
+ len += *in;
+ if (*in++ != 254) break;
+ }
+
+ const u8 * ref = outs + val;
+ const u8 * oe = out + len;
+ if (UNLIKELY(oe > out_end)) oe = out_end;
+
+ while (out < oe) *out++ = *ref++;
+
+ ctx = ((u32)out[-1]) | (((u32)out[-2]) << 8) | (((u32)out[-3]) << 16) | (((u32)out[-4]) << 24);
+ } else {
+ in++;
+ ctx = (ctx << 8) | (*out++ = MATCH);
+ }
+ } else {
+ ctx = (ctx << 8) | (*out++ = *in++);
+ }
+ }
+
+ return out - outs;
+}
+
+static s32 lzp_compress(const u8 * RESTRICT in, u8 * RESTRICT out, s32 n, s32 * RESTRICT lut) {
+ if (n < LZP_MIN_MATCH + 32) return -1;
+
+ memset(lut, 0, sizeof(s32) * (1 << LZP_DICTIONARY));
+
+ return lzp_encode_block(in, in + n, out, out + n, lut);
+}
+
+static s32 lzp_decompress(const u8 * RESTRICT in, u8 * RESTRICT out, s32 n, s32 max, s32 * RESTRICT lut) {
+ if (n < 4) return -1;
+
+ memset(lut, 0, sizeof(s32) * (1 << LZP_DICTIONARY));
+
+ return lzp_decode_block(in, in + n, lut, out, out + max);
+}
+
+/* RLE code. Unlike RLE in other compressors, we collapse all runs if they yield a net gain
+ for a given character and encode this as a set bit in the RLE metadata. This improves the
+ performance and reduces the amount of collapsing done in normal blocks (so that BWT+AC can
+ be more efficient) while we still filter out all the pathological data. */
+
+static s32 mrlec(u8 * in, s32 inlen, u8 * out) {
+ u8 * ip = in;
+ u8 * in_end = in + inlen;
+ s32 op = 0;
+ s32 c, pc = -1;
+ s32 t[256] = { 0 };
+ s32 run = 0;
+ while ((c = (ip < in_end ? *ip++ : -1)) != -1) {
+ if (c == pc)
+ t[c] += (++run % 255) != 0;
+ else
+ --t[c], run = 0;
+ pc = c;
+ }
+ for (s32 i = 0; i < 32; ++i) {
+ c = 0;
+ for (s32 j = 0; j < 8; ++j) c += (t[i * 8 + j] > 0) << j;
+ out[op++] = c;
+ }
+ ip = in;
+ c = pc = -1;
+ run = 0;
+ do {
+ c = ip < in_end ? *ip++ : -1;
+ if (c == pc)
+ ++run;
+ else if (run > 0 && t[pc] > 0) {
+ out[op++] = pc;
+ for (; run > 255; run -= 255) out[op++] = 255;
+ out[op++] = run - 1;
+ run = 1;
+ } else
+ for (++run; run > 1; --run) out[op++] = pc;
+ pc = c;
+ } while (c != -1);
+
+ return op;
+}
+
+static int mrled(u8 * RESTRICT in, u8 * RESTRICT out, s32 outlen, s32 maxin) {
+ s32 op = 0, ip = 0;
+
+ s32 c, pc = -1;
+ s32 t[256] = { 0 };
+ s32 run = 0;
+
+ if (maxin < 32) return 1;
+
+ for (s32 i = 0; i < 32; ++i) {
+ c = in[ip++];
+ for (s32 j = 0; j < 8; ++j) t[i * 8 + j] = (c >> j) & 1;
+ }
+
+ while (op < outlen && ip < maxin) {
+ c = in[ip++];
+ if (t[c]) {
+ for (run = 0; ip < maxin && (pc = in[ip++]) == 255; run += 255)
+ ;
+ run += pc + 1;
+ for (; run > 0 && op < outlen; --run) out[op++] = c;
+ } else
+ out[op++] = c;
+ }
+
+ return op != outlen;
+}
+
+/* The entropy coder. Uses an arithmetic coder implementation outlined in Matt Mahoney's DCE. */
+
+typedef struct {
+ /* Input/output. */
+ u8 *in_queue, *out_queue;
+ s32 input_ptr, output_ptr, input_max;
+
+ /* C0, C1 - used for making the initial prediction, C2 used for an APM with a slightly low
+ learning rate (6) and 512 contexts. kanzi merges C0 and C1, uses slightly different
+ counter initialisation code and prediction code which from my tests tends to be suboptimal. */
+ u16 C0[256], C1[256][256], C2[512][17];
+} state;
+
+#define write_out(s, c) (s)->out_queue[(s)->output_ptr++] = (c)
+#define read_in(s) ((s)->input_ptr < (s)->input_max ? (s)->in_queue[(s)->input_ptr++] : -1)
+
+#define update0(p, x) (p) = ((p) - ((p) >> x))
+#define update1(p, x) (p) = ((p) + (((p) ^ 65535) >> x))
+
+static void begin(state * s) {
+ prefetch(s);
+ for (int i = 0; i < 256; i++) s->C0[i] = 1 << 15;
+ for (int i = 0; i < 256; i++)
+ for (int j = 0; j < 256; j++) s->C1[i][j] = 1 << 15;
+ for (int i = 0; i < 2; i++)
+ for (int j = 0; j < 256; j++)
+ for (int k = 0; k < 17; k++) s->C2[2 * j + i][k] = (k << 12) - (k == 16); // Firm difference from stdpack.
+}
+
+static void encode_bytes(state * s, u8 * buf, s32 size) {
+ /* Arithmetic coding, detecting runs of characters in the file */
+ u32 high = 0xFFFFFFFF, low = 0, c1 = 0, c2 = 0, run = 0;
+
+ for (s32 i = 0; i < size; i++) {
+ u8 c = buf[i];
+
+ if (c1 == c2)
+ ++run;
+ else
+ run = 0;
+
+ const int f = run > 2;
+
+ int ctx = 1;
+
+ while (ctx < 256) {
+ const int p0 = s->C0[ctx];
+ const int p1 = s->C1[c1][ctx];
+ const int p2 = s->C1[c2][ctx];
+ const int p = ((p0 + p1) * 7 + p2 + p2) >> 4;
+
+ const int j = p >> 12;
+ const int x1 = s->C2[2 * ctx + f][j];
+ const int x2 = s->C2[2 * ctx + f][j + 1];
+ const int ssep = x1 + (((x2 - x1) * (p & 4095)) >> 12);
+
+ if (c & 128) {
+ high = low + (((u64)(high - low) * (ssep * 3 + p)) >> 18);
+
+ while ((low ^ high) < (1 << 24)) {
+ write_out(s, low >> 24);
+ low <<= 8;
+ high = (high << 8) + 0xFF;
+ }
+
+ update1(s->C0[ctx], 2);
+ update1(s->C1[c1][ctx], 4);
+ update1(s->C2[2 * ctx + f][j], 6);
+ update1(s->C2[2 * ctx + f][j + 1], 6);
+ ctx += ctx + 1;
+ } else {
+ low += (((u64)(high - low) * (ssep * 3 + p)) >> 18) + 1;
+
+ // Write identical bits.
+ while ((low ^ high) < (1 << 24)) {
+ write_out(s, low >> 24); // Same as high >> 24
+ low <<= 8;
+ high = (high << 8) + 0xFF;
+ }
+
+ update0(s->C0[ctx], 2);
+ update0(s->C1[c1][ctx], 4);
+ update0(s->C2[2 * ctx + f][j], 6);
+ update0(s->C2[2 * ctx + f][j + 1], 6);
+ ctx += ctx;
+ }
+
+ c <<= 1;
+ }
+
+ c2 = c1;
+ c1 = ctx & 255;
+ }
+
+ write_out(s, low >> 24);
+ low <<= 8;
+ write_out(s, low >> 24);
+ low <<= 8;
+ write_out(s, low >> 24);
+ low <<= 8;
+ write_out(s, low >> 24);
+ low <<= 8;
+}
+
+static void decode_bytes(state * s, u8 * c, s32 size) {
+ u32 high = 0xFFFFFFFF, low = 0, c1 = 0, c2 = 0, run = 0, code = 0;
+
+ code = (code << 8) + read_in(s);
+ code = (code << 8) + read_in(s);
+ code = (code << 8) + read_in(s);
+ code = (code << 8) + read_in(s);
+
+ for (s32 i = 0; i < size; i++) {
+ if (c1 == c2)
+ ++run;
+ else
+ run = 0;
+
+ const int f = run > 2;
+
+ int ctx = 1;
+
+ while (ctx < 256) {
+ const int p0 = s->C0[ctx];
+ const int p1 = s->C1[c1][ctx];
+ const int p2 = s->C1[c2][ctx];
+ const int p = ((p0 + p1) * 7 + p2 + p2) >> 4;
+
+ const int j = p >> 12;
+ const int x1 = s->C2[2 * ctx + f][j];
+ const int x2 = s->C2[2 * ctx + f][j + 1];
+ const int ssep = x1 + (((x2 - x1) * (p & 4095)) >> 12);
+
+ const u32 mid = low + (((u64)(high - low) * (ssep * 3 + p)) >> 18);
+ const u8 bit = code <= mid;
+ if (bit)
+ high = mid;
+ else
+ low = mid + 1;
+ while ((low ^ high) < (1 << 24)) {
+ low <<= 8;
+ high = (high << 8) + 255;
+ code = (code << 8) + read_in(s);
+ }
+
+ if (bit) {
+ update1(s->C0[ctx], 2);
+ update1(s->C1[c1][ctx], 4);
+ update1(s->C2[2 * ctx + f][j], 6);
+ update1(s->C2[2 * ctx + f][j + 1], 6);
+ ctx += ctx + 1;
+ } else {
+ update0(s->C0[ctx], 2);
+ update0(s->C1[c1][ctx], 4);
+ update0(s->C2[2 * ctx + f][j], 6);
+ update0(s->C2[2 * ctx + f][j + 1], 6);
+ ctx += ctx;
+ }
+ }
+
+ c2 = c1;
+ c[i] = c1 = ctx & 255;
+ }
+}
+
+/* Public API. */
+
+struct bz3_state {
+ u8 * swap_buffer;
+ s32 block_size;
+ s32 *sais_array, *lzp_lut;
+ state * cm_state;
+ s8 last_error;
+};
+
+BZIP3_API s8 bz3_last_error(struct bz3_state * state) { return state->last_error; }
+
+BZIP3_API const char * bz3_version(void) { return VERSION; }
+
+BZIP3_API size_t bz3_bound(size_t input_size) { return input_size + input_size / 50 + 32; }
+
+BZIP3_API const char * bz3_strerror(struct bz3_state * state) {
+ switch (state->last_error) {
+ case BZ3_OK:
+ return "No error";
+ case BZ3_ERR_OUT_OF_BOUNDS:
+ return "Data index out of bounds";
+ case BZ3_ERR_BWT:
+ return "Burrows-Wheeler transform failed";
+ case BZ3_ERR_CRC:
+ return "CRC32 check failed";
+ case BZ3_ERR_MALFORMED_HEADER:
+ return "Malformed header";
+ case BZ3_ERR_TRUNCATED_DATA:
+ return "Truncated data";
+ case BZ3_ERR_DATA_TOO_BIG:
+ return "Too much data";
+ case BZ3_ERR_DATA_SIZE_TOO_SMALL:
+ return "Size of buffer `buffer_size` passed to the block decoder (bz3_decode_block) is too small. See function docs for details.";
+ default:
+ return "Unknown error";
+ }
+}
+
+BZIP3_API struct bz3_state * bz3_new(s32 block_size) {
+ if (block_size < KiB(65) || block_size > MiB(511)) {
+ return NULL;
+ }
+
+ struct bz3_state * bz3_state = malloc(sizeof(struct bz3_state));
+
+ if (!bz3_state) {
+ return NULL;
+ }
+
+ bz3_state->cm_state = malloc(sizeof(state));
+
+ bz3_state->swap_buffer = malloc(bz3_bound(block_size));
+ bz3_state->sais_array = malloc(BWT_BOUND(block_size) * sizeof(s32));
+ memset(bz3_state->sais_array, 0, sizeof(s32) * BWT_BOUND(block_size));
+
+ bz3_state->lzp_lut = calloc(1 << LZP_DICTIONARY, sizeof(s32));
+
+ if (!bz3_state->cm_state || !bz3_state->swap_buffer || !bz3_state->sais_array || !bz3_state->lzp_lut) {
+ if (bz3_state->cm_state) free(bz3_state->cm_state);
+ if (bz3_state->swap_buffer) free(bz3_state->swap_buffer);
+ if (bz3_state->sais_array) free(bz3_state->sais_array);
+ if (bz3_state->lzp_lut) free(bz3_state->lzp_lut);
+ free(bz3_state);
+ return NULL;
+ }
+
+ bz3_state->block_size = block_size;
+
+ bz3_state->last_error = BZ3_OK;
+
+ return bz3_state;
+}
+
+BZIP3_API void bz3_free(struct bz3_state * state) {
+ free(state->swap_buffer);
+ free(state->sais_array);
+ free(state->cm_state);
+ free(state->lzp_lut);
+ free(state);
+}
+
+#define swap(x, y) \
+ { \
+ u8 * tmp = x; \
+ x = y; \
+ y = tmp; \
+ }
+
+BZIP3_API s32 bz3_encode_block(struct bz3_state * state, u8 * buffer, s32 data_size) {
+ u8 *b1 = buffer, *b2 = state->swap_buffer;
+
+ if (data_size > state->block_size) {
+ state->last_error = BZ3_ERR_DATA_TOO_BIG;
+ return -1;
+ }
+
+ u32 crc32 = crc32sum(1, b1, data_size);
+
+ // Ignore small blocks. They won't benefit from the entropy coding step.
+ if (data_size < 64) {
+ memmove(b1 + 8, b1, data_size);
+ write_neutral_s32(b1, crc32);
+ write_neutral_s32(b1 + 4, -1);
+ return data_size + 8;
+ }
+
+ // Back to front:
+ // bit 1: lzp | no lzp
+ // bit 2: srt | no srt
+ s8 model = 0;
+ s32 lzp_size, rle_size;
+
+ rle_size = mrlec(b1, data_size, b2);
+ if (rle_size < data_size) {
+ swap(b1, b2);
+ data_size = rle_size;
+ model |= 4;
+ }
+
+ lzp_size = lzp_compress(b1, b2, data_size, state->lzp_lut);
+ if (lzp_size > 0 && lzp_size < data_size) {
+ swap(b1, b2);
+ data_size = lzp_size;
+ model |= 2;
+ }
+
+ s32 bwt_idx = libsais_bwt(b1, b2, state->sais_array, data_size, 0, NULL);
+ if (bwt_idx < 0) {
+ state->last_error = BZ3_ERR_BWT;
+ return -1;
+ }
+
+ // Compute the amount of overhead dwords.
+ s32 overhead = 2; // CRC32 + BWT index
+ if (model & 2) overhead++; // LZP
+ if (model & 4) overhead++; // RLE
+
+ begin(state->cm_state);
+ state->cm_state->out_queue = b1 + overhead * 4 + 1;
+ state->cm_state->output_ptr = 0;
+ encode_bytes(state->cm_state, b2, data_size);
+ data_size = state->cm_state->output_ptr;
+
+ // Write the header. Starting with common entries.
+ write_neutral_s32(b1, crc32);
+ write_neutral_s32(b1 + 4, bwt_idx);
+ b1[8] = model;
+
+ s32 p = 0;
+ if (model & 2) write_neutral_s32(b1 + 9 + 4 * p++, lzp_size);
+ if (model & 4) write_neutral_s32(b1 + 9 + 4 * p++, rle_size);
+
+ state->last_error = BZ3_OK;
+
+ if (b1 != buffer) memcpy(buffer, b1, data_size + overhead * 4 + 1);
+
+ return data_size + overhead * 4 + 1;
+}
+
+BZIP3_API s32 bz3_decode_block(struct bz3_state * state, u8 * buffer, size_t buffer_size, s32 compressed_size, s32 orig_size) {
+ // Need minimum bytes for initial header, and compressed_size needs to fit within claimed buffer size.
+ if (buffer_size < 9 || buffer_size < compressed_size) {
+ state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL;
+ return -1;
+ }
+
+ // Read the header.
+ u32 crc32 = read_neutral_s32(buffer);
+ s32 bwt_idx = read_neutral_s32(buffer + 4);
+
+ if (compressed_size > bz3_bound(state->block_size) || compressed_size < 0) {
+ state->last_error = BZ3_ERR_MALFORMED_HEADER;
+ return -1;
+ }
+
+ if (bwt_idx == -1) {
+ if (compressed_size - 8 > 64 || compressed_size < 8) {
+ state->last_error = BZ3_ERR_MALFORMED_HEADER;
+ return -1;
+ }
+
+ // Ensure there's enough space for the raw copied data.
+ if (compressed_size - 8 > buffer_size) {
+ state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL;
+ return -1;
+ }
+
+ memmove(buffer, buffer + 8, compressed_size - 8);
+
+ if (crc32sum(1, buffer, compressed_size - 8) != crc32) {
+ state->last_error = BZ3_ERR_CRC;
+ return -1;
+ }
+
+ return compressed_size - 8;
+ }
+
+ s8 model = buffer[8];
+
+ // Ensure we have sufficient bytes for the rle/lzp sizes.
+ size_t needed_header_size = 9 + ((model & 2) * 4) + ((model & 4) * 4);
+ if (buffer_size < needed_header_size) {
+ state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL;
+ return -1;
+ }
+
+ s32 lzp_size = -1, rle_size = -1, p = 0;
+ if (model & 2) lzp_size = read_neutral_s32(buffer + 9 + 4 * p++);
+ if (model & 4) rle_size = read_neutral_s32(buffer + 9 + 4 * p++);
+ p += 2;
+
+ compressed_size -= p * 4 + 1;
+
+ if (((model & 2) && (lzp_size > bz3_bound(state->block_size) || lzp_size < 0)) ||
+ ((model & 4) && (rle_size > bz3_bound(state->block_size) || rle_size < 0))) {
+ state->last_error = BZ3_ERR_MALFORMED_HEADER;
+ return -1;
+ }
+
+ if (orig_size > bz3_bound(state->block_size) || orig_size < 0) {
+ state->last_error = BZ3_ERR_MALFORMED_HEADER;
+ return -1;
+ }
+
+ // Size that undoing BWT+BCM should decompress into.
+ s32 size_before_bwt;
+
+ if (model & 2)
+ size_before_bwt = lzp_size;
+ else if (model & 4)
+ size_before_bwt = rle_size;
+ else
+ size_before_bwt = orig_size;
+
+ // Note(sewer): It's technically valid within the spec to create a bzip3 block
+ // where the size after LZP/RLE is larger than the original input. Some earlier encoders
+ // even (mistakenly?) were able to do this.
+ if (!bz3_check_buffer_size(buffer_size, lzp_size, rle_size, orig_size)) {
+ state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL;
+ return -1;
+ }
+
+ // Decode the data.
+ u8 *b1 = buffer, *b2 = state->swap_buffer;
+
+ begin(state->cm_state);
+ state->cm_state->in_queue = b1 + p * 4 + 1;
+ state->cm_state->input_ptr = 0;
+ state->cm_state->input_max = compressed_size;
+
+ decode_bytes(state->cm_state, b2, size_before_bwt);
+ swap(b1, b2);
+
+ if (bwt_idx > size_before_bwt) {
+ state->last_error = BZ3_ERR_MALFORMED_HEADER;
+ return -1;
+ }
+
+ // Undo BWT
+ memset(state->sais_array, 0, sizeof(s32) * BWT_BOUND(state->block_size));
+ memset(b2, 0, size_before_bwt); // buffer b2, swap b1
+ if (libsais_unbwt(b1, b2, state->sais_array, size_before_bwt, NULL, bwt_idx) < 0) {
+ state->last_error = BZ3_ERR_BWT;
+ return -1;
+ }
+ swap(b1, b2);
+
+ s32 size_src = size_before_bwt;
+
+ // Undo LZP
+ if (model & 2) {
+ size_src = lzp_decompress(b1, b2, lzp_size, bz3_bound(state->block_size), state->lzp_lut);
+ if (size_src == -1) {
+ state->last_error = BZ3_ERR_CRC;
+ return -1;
+ }
+ // SAFETY(sewer): An attacker formed bzip3 data which decompresses as valid lzp.
+ // The headers above were set to ones that pass validation (size within bounds), but the
+ // data itself tries to escape buffer_size. Don't allow it to.
+ if (size_src > buffer_size) {
+ state->last_error = BZ3_ERR_DATA_SIZE_TOO_SMALL;
+ return -1;
+ }
+ swap(b1, b2);
+ }
+
+ if (model & 4) {
+ // SAFETY: mrled is capped at orig_size, which is in bounds.
+ int err = mrled(b1, b2, orig_size, size_src);
+ if (err) {
+ state->last_error = BZ3_ERR_CRC;
+ return -1;
+ }
+ size_src = orig_size;
+ swap(b1, b2);
+ }
+
+ state->last_error = BZ3_OK;
+
+ if (size_src > state->block_size || size_src < 0) {
+ state->last_error = BZ3_ERR_MALFORMED_HEADER;
+ return -1;
+ }
+
+ if (b1 != buffer) memcpy(buffer, b1, size_src);
+
+ if (crc32 != crc32sum(1, buffer, size_src)) {
+ state->last_error = BZ3_ERR_CRC;
+ return -1;
+ }
+
+ return size_src;
+}
+
+#undef swap
+
+#ifdef PTHREAD
+
+ #include
+
+typedef struct {
+ struct bz3_state * state;
+ u8 * buffer;
+ s32 size;
+} encode_thread_msg;
+
+typedef struct {
+ struct bz3_state * state;
+ u8 * buffer;
+ size_t buffer_size;
+ s32 size;
+ s32 orig_size;
+} decode_thread_msg;
+
+static void * bz3_init_encode_thread(void * _msg) {
+ encode_thread_msg * msg = _msg;
+ msg->size = bz3_encode_block(msg->state, msg->buffer, msg->size);
+ pthread_exit(NULL);
+ return NULL; // unreachable
+}
+
+static void * bz3_init_decode_thread(void * _msg) {
+ decode_thread_msg * msg = _msg;
+ bz3_decode_block(msg->state, msg->buffer, msg->buffer_size, msg->size, msg->orig_size);
+ pthread_exit(NULL);
+ return NULL; // unreachable
+}
+
+BZIP3_API void bz3_encode_blocks(struct bz3_state * states[], u8 * buffers[], s32 sizes[], s32 n) {
+ encode_thread_msg messages[n];
+ pthread_t threads[n];
+ for (s32 i = 0; i < n; i++) {
+ messages[i].state = states[i];
+ messages[i].buffer = buffers[i];
+ messages[i].size = sizes[i];
+ pthread_create(&threads[i], NULL, bz3_init_encode_thread, &messages[i]);
+ }
+ for (s32 i = 0; i < n; i++) pthread_join(threads[i], NULL);
+ for (s32 i = 0; i < n; i++) sizes[i] = messages[i].size;
+}
+
+BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], u8 * buffers[], size_t buffer_sizes[], s32 sizes[], s32 orig_sizes[], s32 n) {
+ decode_thread_msg messages[n];
+ pthread_t threads[n];
+ for (s32 i = 0; i < n; i++) {
+ messages[i].state = states[i];
+ messages[i].buffer = buffers[i];
+ messages[i].buffer_size = buffer_sizes[i];
+ messages[i].size = sizes[i];
+ messages[i].orig_size = orig_sizes[i];
+ pthread_create(&threads[i], NULL, bz3_init_decode_thread, &messages[i]);
+ }
+ for (s32 i = 0; i < n; i++) pthread_join(threads[i], NULL);
+}
+
+#endif
+
+/* High level API implementations. */
+
+BZIP3_API int bz3_compress(u32 block_size, const u8 * const in, u8 * out, size_t in_size, size_t * out_size) {
+ if (block_size > in_size) block_size = in_size + 16;
+ block_size = block_size <= KiB(65) ? KiB(65) : block_size;
+
+ struct bz3_state * state = bz3_new(block_size);
+ if (!state) return BZ3_ERR_INIT;
+
+ u8 * compression_buf = malloc(bz3_bound(block_size));
+ if (!compression_buf) {
+ bz3_free(state);
+ return BZ3_ERR_INIT;
+ }
+
+ size_t buf_max = *out_size;
+ *out_size = 0;
+
+ u32 n_blocks = in_size / block_size;
+ if (in_size % block_size) n_blocks++;
+
+ if (buf_max < 13 || buf_max < bz3_bound(in_size)) {
+ bz3_free(state);
+ free(compression_buf);
+ return BZ3_ERR_DATA_TOO_BIG;
+ }
+
+ out[0] = 'B';
+ out[1] = 'Z';
+ out[2] = '3';
+ out[3] = 'v';
+ out[4] = '1';
+ write_neutral_s32(out + 5, block_size);
+ write_neutral_s32(out + 9, n_blocks);
+ *out_size += 13;
+
+ // Compress and write the blocks.
+ size_t in_offset = 0;
+ for (u32 i = 0; i < n_blocks; i++) {
+ s32 size = block_size;
+ if (i == n_blocks - 1) size = in_size % block_size;
+ memcpy(compression_buf, in + in_offset, size);
+ s32 out_size_block = bz3_encode_block(state, compression_buf, size);
+ if (bz3_last_error(state) != BZ3_OK) {
+ s8 last_error = state->last_error;
+ bz3_free(state);
+ free(compression_buf);
+ return last_error;
+ }
+ memcpy(out + *out_size + 8, compression_buf, out_size_block);
+ write_neutral_s32(out + *out_size, out_size_block);
+ write_neutral_s32(out + *out_size + 4, size);
+ *out_size += out_size_block + 8;
+ in_offset += size;
+ }
+
+ bz3_free(state);
+ free(compression_buf);
+ return BZ3_OK;
+}
+
+BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, size_t * out_size) {
+ if (in_size < 13) return BZ3_ERR_MALFORMED_HEADER;
+ if (in[0] != 'B' || in[1] != 'Z' || in[2] != '3' || in[3] != 'v' || in[4] != '1') {
+ return BZ3_ERR_MALFORMED_HEADER;
+ }
+ u32 block_size = read_neutral_s32(in + 5);
+ u32 n_blocks = read_neutral_s32(in + 9);
+ in_size -= 13;
+ in += 13;
+
+ struct bz3_state * state = bz3_new(block_size);
+ if (!state) return BZ3_ERR_INIT;
+
+ size_t compression_buf_size = bz3_bound(block_size);
+ u8 * compression_buf = malloc(compression_buf_size);
+ if (!compression_buf) {
+ bz3_free(state);
+ return BZ3_ERR_INIT;
+ }
+
+ size_t buf_max = *out_size;
+ *out_size = 0;
+
+ for (u32 i = 0; i < n_blocks; i++) {
+ if (in_size < 8) {
+ malformed_header:
+ bz3_free(state);
+ free(compression_buf);
+ return BZ3_ERR_MALFORMED_HEADER;
+ }
+ s32 size = read_neutral_s32(in);
+ if (size < 0 || size > block_size) goto malformed_header;
+ if (in_size < size + 8) {
+ bz3_free(state);
+ free(compression_buf);
+ return BZ3_ERR_TRUNCATED_DATA;
+ }
+ s32 orig_size = read_neutral_s32(in + 4);
+ if (orig_size < 0) goto malformed_header;
+ if (buf_max < *out_size + orig_size) {
+ bz3_free(state);
+ free(compression_buf);
+ return BZ3_ERR_DATA_TOO_BIG;
+ }
+ memcpy(compression_buf, in + 8, size);
+ bz3_decode_block(state, compression_buf, compression_buf_size, size, orig_size);
+ if (bz3_last_error(state) != BZ3_OK) {
+ s8 last_error = state->last_error;
+ bz3_free(state);
+ free(compression_buf);
+ return last_error;
+ }
+ memcpy(out + *out_size, compression_buf, orig_size);
+ *out_size += orig_size;
+ in += size + 8;
+ in_size -= size + 8;
+ }
+
+ bz3_free(state);
+ return BZ3_OK;
+}
+
+BZIP3_API size_t bz3_min_memory_needed(int32_t block_size) {
+ if (block_size < KiB(65) || block_size > MiB(511)) {
+ return 0;
+ }
+
+ size_t total_size = 0;
+
+ // This is based on bz3_new.
+ // Core state structure
+ total_size += sizeof(struct bz3_state);
+
+ // cm_state
+ total_size += sizeof(state);
+
+ // Swap buffer (needs to handle expanded size) (swap_buffer)
+ total_size += bz3_bound(block_size);
+
+ // SAIS array
+ total_size += BWT_BOUND(block_size) * sizeof(int32_t);
+
+ // LZP lookup table (lzp_lut)
+ total_size += (1 << LZP_DICTIONARY) * sizeof(int32_t);
+ return total_size;
+}
+
+
+BZIP3_API int bz3_orig_size_sufficient_for_decode(const u8 * block, size_t block_size, s32 orig_size) {
+ // Need at least 9 bytes for the initial header (4 bytes BWT index + 4 bytes CRC + 1 byte model)
+ if (block_size < 9) {
+ return -1;
+ }
+
+ s32 bwt_idx = read_neutral_s32(block + 4);
+ if (bwt_idx == -1) {
+ // Uncompressed literals.
+ // Original size always sufficient for uncompressed blocks
+ return 1;
+ }
+
+ s8 model = block[8];
+ s32 lzp_size = -1, rle_size = -1;
+ size_t header_size = 9; // Start after model byte
+
+ // Ensure we have sufficient bytes for the rle/lzp sizes.
+ size_t needed_header_size = 9 + ((model & 2) * 4) + ((model & 4) * 4);
+ if (block_size < needed_header_size) {
+ return -1;
+ }
+
+ // Need additional 4 bytes for each size field that might be present
+ if (model & 2) {
+ lzp_size = read_neutral_s32(block + header_size);
+ header_size += 4;
+ }
+ if (model & 4) rle_size = read_neutral_s32(block + header_size);
+ return bz3_check_buffer_size((size_t)orig_size, lzp_size, rle_size, orig_size);
+}
diff --git a/include/common.h b/include/common.h
new file mode 100644
--- /dev/null
+++ b/include/common.h
@@ -0,0 +1,126 @@
+
+/*
+ * BZip3 - A spiritual successor to BZip2.
+ * Copyright (C) 2022-2024 Kamila Szewczyk
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License along with
+ * this program. If not, see .
+ */
+
+#ifndef _COMMON_H
+#define _COMMON_H
+
+#define KiB(x) ((x)*1024)
+#define MiB(x) ((x)*1024 * 1024)
+#define BWT_BOUND(x) (bz3_bound(x) + 128)
+
+#include
+#include
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+
+typedef int8_t s8;
+typedef int16_t s16;
+typedef int32_t s32;
+
+static s32 read_neutral_s32(const u8 * data) {
+ return ((u32)data[0]) | (((u32)data[1]) << 8) | (((u32)data[2]) << 16) | (((u32)data[3]) << 24);
+}
+
+static void write_neutral_s32(u8 * data, s32 value) {
+ data[0] = value & 0xFF;
+ data[1] = (value >> 8) & 0xFF;
+ data[2] = (value >> 16) & 0xFF;
+ data[3] = (value >> 24) & 0xFF;
+}
+
+#if defined(__GNUC__) || defined(__clang__)
+ #define RESTRICT __restrict__
+#elif defined(_MSC_VER) || defined(__INTEL_COMPILER)
+ #define RESTRICT __restrict
+#else
+ #define RESTRICT restrict
+ #warning Your compiler, configuration or platform might not be supported.
+#endif
+
+#if defined(__has_builtin)
+ #if __has_builtin(__builtin_prefetch)
+ #define HAS_BUILTIN_PREFETCH
+ #endif
+#elif defined(__GNUC__) && (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 2)) || (__GNUC__ >= 4))
+ #define HAS_BUILTIN_PREFETCH
+#endif
+
+#if defined(__has_builtin)
+ #if __has_builtin(__builtin_bswap16)
+ #define HAS_BUILTIN_BSWAP16
+ #endif
+#elif defined(__GNUC__) && (((__GNUC__ == 4) && (__GNUC_MINOR__ >= 8)) || (__GNUC__ >= 5))
+ #define HAS_BUILTIN_BSWAP16
+#endif
+
+#if defined(HAS_BUILTIN_PREFETCH)
+ #define prefetch(address) __builtin_prefetch((const void *)(address), 0, 0)
+ #define prefetchw(address) __builtin_prefetch((const void *)(address), 1, 0)
+#elif defined(_M_IX86) || defined(_M_AMD64) || defined(__x86_64__) || defined(i386) || defined(__i386__) || \
+ defined(__i386)
+ #include
+ #define prefetch(address) _mm_prefetch((const void *)(address), _MM_HINT_NTA)
+ #define prefetchw(address) _m_prefetchw((const void *)(address))
+#elif defined(_M_ARM) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || \
+ defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__)
+ #include
+ #define prefetch(address) __prefetch((const void *)(address))
+ #define prefetchw(address) __prefetchw((const void *)(address))
+#elif defined(_M_ARM64) || defined(__aarch64__)
+ #include
+ #define prefetch(address) __prefetch2((const void *)(address), 1)
+ #define prefetchw(address) __prefetch2((const void *)(address), 17)
+#else
+ #error Your compiler, configuration or platform is not supported.
+#endif
+
+#if !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+ #if defined(_LITTLE_ENDIAN) || (defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && BYTE_ORDER == LITTLE_ENDIAN) || \
+ (defined(_BYTE_ORDER) && defined(_LITTLE_ENDIAN) && _BYTE_ORDER == _LITTLE_ENDIAN) || \
+ (defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && __BYTE_ORDER == __LITTLE_ENDIAN) || \
+ (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+ #define __LITTLE_ENDIAN__
+ #elif defined(_BIG_ENDIAN) || (defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN) || \
+ (defined(_BYTE_ORDER) && defined(_BIG_ENDIAN) && _BYTE_ORDER == _BIG_ENDIAN) || \
+ (defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && __BYTE_ORDER == __BIG_ENDIAN) || \
+ (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+ #define __BIG_ENDIAN__
+ #elif defined(_WIN32)
+ #define __LITTLE_ENDIAN__
+ #endif
+#endif
+
+#if defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+ #if defined(HAS_BUILTIN_BSWAP16)
+ #define bswap16(x) (__builtin_bswap16(x))
+ #elif defined(_MSC_VER) && !defined(__INTEL_COMPILER)
+ #define bswap16(x) (_byteswap_ushort(x))
+ #else
+ #define bswap16(x) ((u16)(x >> 8) | (u16)(x << 8))
+ #endif
+#elif !defined(__LITTLE_ENDIAN__) && defined(__BIG_ENDIAN__)
+ #define bswap16(x) (x)
+#else
+ #error Your compiler, configuration or platform is not supported.
+#endif
+
+#endif
diff --git a/include/libbz3.h b/include/libbz3.h
new file mode 100644
--- /dev/null
+++ b/include/libbz3.h
@@ -0,0 +1,242 @@
+
+/*
+ * BZip3 - A spiritual successor to BZip2.
+ * Copyright (C) 2022-2024 Kamila Szewczyk
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License along with
+ * this program. If not, see .
+ */
+
+#ifndef LIBBZ3_H
+#define LIBBZ3_H
+
+#include
+#include
+
+/* Symbol visibility control. */
+#ifndef BZIP3_VISIBLE
+ #if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__)
+ #define BZIP3_VISIBLE __attribute__((visibility("default")))
+ #else
+ #define BZIP3_VISIBLE
+ #endif
+#endif
+
+#if defined(BZIP3_DLL_EXPORT) && (BZIP3_DLL_EXPORT == 1)
+ #define BZIP3_API __declspec(dllexport) BZIP3_VISIBLE
+#elif defined(BZIP3_DLL_IMPORT) && (BZIP3_DLL_IMPORT == 1)
+ #define BZIP3_API __declspec(dllimport) BZIP3_VISIBLE
+#else
+ #define BZIP3_API BZIP3_VISIBLE
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define BZ3_OK 0
+#define BZ3_ERR_OUT_OF_BOUNDS -1
+#define BZ3_ERR_BWT -2
+#define BZ3_ERR_CRC -3
+#define BZ3_ERR_MALFORMED_HEADER -4
+#define BZ3_ERR_TRUNCATED_DATA -5
+#define BZ3_ERR_DATA_TOO_BIG -6
+#define BZ3_ERR_INIT -7
+#define BZ3_ERR_DATA_SIZE_TOO_SMALL -8
+
+struct bz3_state;
+
+/**
+ * @brief Get bzip3 version.
+ */
+BZIP3_API const char * bz3_version(void);
+
+/**
+ * @brief Get the last error number associated with a given state.
+ */
+BZIP3_API int8_t bz3_last_error(struct bz3_state * state);
+
+/**
+ * @brief Return a user-readable message explaining the cause of the last error.
+ */
+BZIP3_API const char * bz3_strerror(struct bz3_state * state);
+
+/**
+ * @brief Construct a new block encoder state, which will encode blocks as big as the given block size.
+ * The decoder will be able to decode blocks at most as big as the given block size.
+ * Returns NULL in case allocation fails or the block size is not between 65K and 511M
+ */
+BZIP3_API struct bz3_state * bz3_new(int32_t block_size);
+
+/**
+ * @brief Free the memory occupied by a block encoder state.
+ */
+BZIP3_API void bz3_free(struct bz3_state * state);
+
+/**
+ * @brief Return the recommended size of the output buffer for the compression functions.
+ */
+BZIP3_API size_t bz3_bound(size_t input_size);
+
+/* ** HIGH LEVEL APIs ** */
+
+/**
+ * @brief Compress a frame. This function does not support parallelism
+ * by itself, consider using the low level `bz3_encode_blocks()` function instead.
+ * Using the low level API might provide better performance.
+ * Returns a bzip3 error code; BZ3_OK when the operation is successful.
+ * Make sure to set out_size to the size of the output buffer before the operation;
+ * out_size must be at least equal to `bz3_bound(in_size)'.
+ */
+BZIP3_API int bz3_compress(uint32_t block_size, const uint8_t * in, uint8_t * out, size_t in_size, size_t * out_size);
+
+/**
+ * @brief Decompress a frame. This function does not support parallelism
+ * by itself, consider using the low level `bz3_decode_blocks()` function instead.
+ * Using the low level API might provide better performance.
+ * Returns a bzip3 error code; BZ3_OK when the operation is successful.
+ * Make sure to set out_size to the size of the output buffer before the operation.
+ */
+BZIP3_API int bz3_decompress(const uint8_t * in, uint8_t * out, size_t in_size, size_t * out_size);
+
+/**
+ * @brief Calculate the minimal memory required for compression with the given block size.
+ * This includes all internal buffers and state structures. This calculates the amount of bytes
+ * that will be allocated by a call to `bz3_new()`.
+ *
+ * @details Memory allocation and usage patterns:
+ *
+ * bz3_new():
+ * - Allocates all memory upfront:
+ * - Core state structure (sizeof(struct bz3_state))
+ * - Swap buffer (bz3_bound(block_size) bytes)
+ * - SAIS array (BWT_BOUND(block_size) * sizeof(int32_t) bytes)
+ * - LZP lookup table ((1 << LZP_DICTIONARY) * sizeof(int32_t) bytes)
+ * - Compression state (sizeof(state))
+ * - All memory remains allocated until bz3_free()
+ *
+ * Additional memory may be used depending on API used from here.
+ *
+ * # Low Level APIs
+ *
+ * 1. bz3_encode_block() / bz3_decode_block():
+ * - Uses pre-allocated memory from bz3_new()
+ * - No additional memory allocation except for libsais (usually ~16KiB)
+ * - Peak memory usage of physical RAM varies with compression stages:
+ * - LZP: Uses LZP lookup table + swap buffer
+ * - BWT: Uses SAIS array + swap buffer
+ * - Entropy coding: Uses compression state (cm_state) + swap buffer
+ *
+ * Using the higher level API, `bz3_compress`, expect an additional allocation
+ * of `bz3_bound(block_size)`.
+ *
+ * In the parallel version `bz3_encode_blocks`, each thread gets its own state,
+ * so memory usage is `n_threads * bz3_compress_memory_needed()`.
+ *
+ * # High Level APIs
+ *
+ * 1. bz3_compress():
+ * - Allocates additional temporary compression buffer (bz3_bound(block_size) bytes)
+ * in addition to the memory amount returned by this method call and libsais.
+ * - Everything is freed after compression completes
+ *
+ * 2. bz3_decompress():
+ * - Allocates additional temporary compression buffer (bz3_bound(block_size) bytes)
+ * in addition to the memory amount returned by this method call and libsais.
+ * - Everything is freed after compression completes
+ *
+ * Memory remains constant during operation, with except of some small allocations from libsais during
+ * BWT stage. That is not accounted by this function, though it usually amounts to ~16KiB, negligible.
+ * The worst case of BWT is 2*block_size technically speaking.
+ *
+ * No dynamic (re)allocation occurs outside of that.
+ *
+ * @param block_size The block size to be used for compression
+ * @return The total number of bytes required for compression, or 0 if block_size is invalid
+ */
+BZIP3_API size_t bz3_min_memory_needed(int32_t block_size);
+
+/* ** LOW LEVEL APIs ** */
+
+/**
+ * @brief Encode a single block. Returns the amount of bytes written to `buffer'.
+ * `buffer' must be able to hold at least `bz3_bound(size)' bytes. The size must not
+ * exceed the block size associated with the state.
+ */
+BZIP3_API int32_t bz3_encode_block(struct bz3_state * state, uint8_t * buffer, int32_t size);
+
+/**
+ * @brief Decode a single block.
+ *
+ * `buffer' must be able to hold at least `bz3_bound(orig_size)' bytes
+ * in order to ensure decompression will succeed for all possible bzip3 blocks.
+ *
+ * In most (but not all) cases, `orig_size` should usually be sufficient.
+ * If it is not sufficient, you must allocate a buffer of size `bz3_bound(orig_size)` temporarily.
+ *
+ * If `buffer_size` is too small, `BZ3_ERR_DATA_SIZE_TOO_SMALL` will be returned.
+ * The size must not exceed the block size associated with the state.
+ *
+ * @param buffer_size The size of the buffer at 'buffer'
+ * @param compressed_size The size of the compressed data in 'buffer'
+ * @param orig_size The original size of the data before compression.
+ */
+BZIP3_API int32_t bz3_decode_block(struct bz3_state * state, uint8_t * buffer, size_t buffer_size, int32_t compressed_size, int32_t orig_size);
+
+/**
+ * @brief Encode `n' blocks, all in parallel.
+ * All specifics of the `bz3_encode_block' still hold. The function will launch a thread for each block.
+ * The compressed sizes are written to the `sizes' array. Every buffer is overwritten and none of them can overlap.
+ * Precisely `n' states, buffers and sizes must be supplied.
+ *
+ * Expects `n' between 2 and 16.
+ *
+ * Present in the shared library only if -lpthread was present during building.
+ */
+BZIP3_API void bz3_encode_blocks(struct bz3_state * states[], uint8_t * buffers[], int32_t sizes[], int32_t n);
+
+/**
+ * @brief Decode `n' blocks, all in parallel.
+ * Same specifics as `bz3_encode_blocks', but doesn't overwrite `sizes'.
+ */
+BZIP3_API void bz3_decode_blocks(struct bz3_state * states[], uint8_t * buffers[], size_t buffer_sizes[], int32_t sizes[],
+ int32_t orig_sizes[], int32_t n);
+
+/**
+ * @brief Check if using original file size as buffer size is sufficient for decompressing
+ * a block at `block` pointer.
+ *
+ * @param block Pointer to the compressed block data
+ * @param block_size Size of the block buffer in bytes (must be at least 13 bytes for header)
+ * @param orig_size Size of the original uncompressed data
+ * @return 1 if original size is sufficient, 0 if insufficient, -1 on header error (insufficient buffer size)
+ *
+ * @remarks
+ *
+ * This function is useful for external APIs using the low level block encoding API,
+ * `bz3_encode_block`. You would normally call this directly after `bz3_encode_block`
+ * on the block that has been output.
+ *
+ * The purpose of this function is to prevent encoding blocks that would require an additional
+ * malloc at decompress time.
+ * The goal is to prevent erroring with `BZ3_ERR_DATA_SIZE_TOO_SMALL`, thus
+ * in turn
+ */
+BZIP3_API int bz3_orig_size_sufficient_for_decode(const uint8_t * block, size_t block_size, int32_t orig_size);
+
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
diff --git a/include/libsais.h b/include/libsais.h
new file mode 100644
--- /dev/null
+++ b/include/libsais.h
@@ -0,0 +1,5428 @@
+/*--
+
+This file is a part of libsais, a library for linear time suffix array,
+longest common prefix array and burrows wheeler transform construction.
+
+ Copyright (c) 2021-2022 Ilya Grebnov
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+Please see the file LICENSE for full copyright information.
+
+The stability patches that fix undefined behaviour in unbwt routines:
+
+ Copyright (c) 2022 Kamila Szewczyk
+
+ Licensed under the same license as the original software.
+
+--*/
+
+#ifndef LIBSAIS_H
+#define LIBSAIS_H
+
+#include "common.h"
+
+/* libsais source code amalgamate. */
+
+#include
+#include
+#include
+#include
+#include
+
+#define UNUSED(_x) (void)(_x)
+
+typedef s32 sa_sint_t;
+typedef u32 sa_uint_t;
+typedef ptrdiff_t fast_sint_t;
+typedef size_t fast_uint_t;
+
+#define SAINT_BIT (32)
+#define SAINT_MAX INT32_MAX
+#define SAINT_MIN INT32_MIN
+
+#define ALPHABET_SIZE (1 << CHAR_BIT)
+#define UNBWT_FASTBITS (17)
+
+#define SUFFIX_GROUP_BIT (SAINT_BIT - 1)
+#define SUFFIX_GROUP_MARKER (((sa_sint_t)1) << (SUFFIX_GROUP_BIT - 1))
+
+#define BUCKETS_INDEX2(_c, _s) (((_c) << 1) + (_s))
+#define BUCKETS_INDEX4(_c, _s) (((_c) << 2) + (_s))
+
+#define LIBSAIS_PER_THREAD_CACHE_SIZE (24576)
+
+typedef struct LIBSAIS_THREAD_CACHE {
+ sa_sint_t symbol;
+ sa_sint_t index;
+} LIBSAIS_THREAD_CACHE;
+
+typedef union LIBSAIS_THREAD_STATE {
+ struct {
+ fast_sint_t position;
+ fast_sint_t count;
+
+ fast_sint_t m;
+ fast_sint_t last_lms_suffix;
+
+ sa_sint_t * buckets;
+ LIBSAIS_THREAD_CACHE * cache;
+ } state;
+
+ u8 padding[64];
+} LIBSAIS_THREAD_STATE;
+
+typedef struct LIBSAIS_CONTEXT {
+ sa_sint_t * buckets;
+ LIBSAIS_THREAD_STATE * thread_state;
+ fast_sint_t threads;
+} LIBSAIS_CONTEXT;
+
+typedef struct LIBSAIS_UNBWT_CONTEXT {
+ sa_uint_t * bucket2;
+ u16 * fastbits;
+ sa_uint_t * buckets;
+ fast_sint_t threads;
+} LIBSAIS_UNBWT_CONTEXT;
+
+static void * libsais_align_up(const void * address, size_t alignment) {
+ return (void *)((((ptrdiff_t)address) + ((ptrdiff_t)alignment) - 1) & (-((ptrdiff_t)alignment)));
+}
+
+static void * libsais_alloc_aligned(size_t size, size_t alignment) {
+ void * address = malloc(size + sizeof(short) + alignment - 1);
+ if (address != NULL) {
+ void * aligned_address = libsais_align_up((void *)((ptrdiff_t)address + (ptrdiff_t)(sizeof(short))), alignment);
+ ((short *)aligned_address)[-1] = (short)((ptrdiff_t)aligned_address - (ptrdiff_t)address);
+
+ return aligned_address;
+ }
+
+ return NULL;
+}
+
+static void libsais_free_aligned(void * aligned_address) {
+ if (aligned_address != NULL) {
+ free((void *)((ptrdiff_t)aligned_address - ((short *)aligned_address)[-1]));
+ }
+}
+
+static LIBSAIS_THREAD_STATE * libsais_alloc_thread_state(sa_sint_t threads) {
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state =
+ (LIBSAIS_THREAD_STATE *)libsais_alloc_aligned((size_t)threads * sizeof(LIBSAIS_THREAD_STATE), 4096);
+ sa_sint_t * RESTRICT thread_buckets =
+ (sa_sint_t *)libsais_alloc_aligned((size_t)threads * 4 * ALPHABET_SIZE * sizeof(sa_sint_t), 4096);
+ LIBSAIS_THREAD_CACHE * RESTRICT thread_cache = (LIBSAIS_THREAD_CACHE *)libsais_alloc_aligned(
+ (size_t)threads * LIBSAIS_PER_THREAD_CACHE_SIZE * sizeof(LIBSAIS_THREAD_CACHE), 4096);
+
+ if (thread_state != NULL && thread_buckets != NULL && thread_cache != NULL) {
+ fast_sint_t t;
+ for (t = 0; t < threads; ++t) {
+ thread_state[t].state.buckets = thread_buckets;
+ thread_buckets += 4 * ALPHABET_SIZE;
+ thread_state[t].state.cache = thread_cache;
+ thread_cache += LIBSAIS_PER_THREAD_CACHE_SIZE;
+ }
+
+ return thread_state;
+ }
+
+ libsais_free_aligned(thread_cache);
+ libsais_free_aligned(thread_buckets);
+ libsais_free_aligned(thread_state);
+ return NULL;
+}
+
+static void libsais_free_thread_state(LIBSAIS_THREAD_STATE * thread_state) {
+ if (thread_state != NULL) {
+ libsais_free_aligned(thread_state[0].state.cache);
+ libsais_free_aligned(thread_state[0].state.buckets);
+ libsais_free_aligned(thread_state);
+ }
+}
+
+static LIBSAIS_CONTEXT * libsais_create_ctx_main(sa_sint_t threads) {
+ LIBSAIS_CONTEXT * RESTRICT ctx = (LIBSAIS_CONTEXT *)libsais_alloc_aligned(sizeof(LIBSAIS_CONTEXT), 64);
+ sa_sint_t * RESTRICT buckets = (sa_sint_t *)libsais_alloc_aligned(8 * ALPHABET_SIZE * sizeof(sa_sint_t), 4096);
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state = threads > 1 ? libsais_alloc_thread_state(threads) : NULL;
+
+ if (ctx != NULL && buckets != NULL && (thread_state != NULL || threads == 1)) {
+ ctx->buckets = buckets;
+ ctx->threads = threads;
+ ctx->thread_state = thread_state;
+
+ return ctx;
+ }
+
+ libsais_free_thread_state(thread_state);
+ libsais_free_aligned(buckets);
+ libsais_free_aligned(ctx);
+ return NULL;
+}
+
+static void libsais_free_ctx_main(LIBSAIS_CONTEXT * ctx) {
+ if (ctx != NULL) {
+ libsais_free_thread_state(ctx->thread_state);
+ libsais_free_aligned(ctx->buckets);
+ libsais_free_aligned(ctx);
+ }
+}
+static void libsais_gather_lms_suffixes_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n, fast_sint_t m,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ if (omp_block_size > 0) {
+ const fast_sint_t prefetch_distance = 128;
+
+ fast_sint_t i, j = omp_block_start + omp_block_size, c0 = T[omp_block_start + omp_block_size - 1], c1 = -1;
+
+ while (j < n && (c1 = T[j]) == c0) {
+ ++j;
+ }
+
+ fast_uint_t s = c0 >= c1;
+
+ for (i = omp_block_start + omp_block_size - 2, j = omp_block_start + 3; i >= j; i -= 4) {
+ prefetch(&T[i - prefetch_distance]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 0);
+ m -= ((s & 3) == 1);
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 1);
+ m -= ((s & 3) == 1);
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 2);
+ m -= ((s & 3) == 1);
+ }
+
+ for (j -= 3; i >= j; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ }
+
+ SA[m] = (sa_sint_t)(i + 1);
+ }
+}
+
+static void libsais_gather_lms_suffixes_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t threads, LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (n / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : n - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ libsais_gather_lms_suffixes_8u(T, SA, n, (fast_sint_t)n - 1, omp_block_start, omp_block_size);
+ }
+ }
+}
+
+static sa_sint_t libsais_gather_lms_suffixes_32s(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t i = n - 2;
+ sa_sint_t m = n - 1;
+ fast_uint_t s = 1;
+ fast_sint_t c0 = T[n - 1];
+ fast_sint_t c1 = 0;
+
+ for (; i >= 3; i -= 4) {
+ prefetch(&T[i - prefetch_distance]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = i + 1;
+ m -= ((s & 3) == 1);
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = i - 0;
+ m -= ((s & 3) == 1);
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = i - 1;
+ m -= ((s & 3) == 1);
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = i - 2;
+ m -= ((s & 3) == 1);
+ }
+
+ for (; i >= 0; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = i + 1;
+ m -= ((s & 3) == 1);
+ }
+
+ return n - 1 - m;
+}
+
+static sa_sint_t libsais_gather_compacted_lms_suffixes_32s(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t i = n - 2;
+ sa_sint_t m = n - 1;
+ fast_uint_t s = 1;
+ fast_sint_t c0 = T[n - 1];
+ fast_sint_t c1 = 0;
+
+ for (; i >= 3; i -= 4) {
+ prefetch(&T[i - prefetch_distance]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = i + 1;
+ m -= ((fast_sint_t)(s & 3) == (c0 >= 0));
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = i - 0;
+ m -= ((fast_sint_t)(s & 3) == (c1 >= 0));
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = i - 1;
+ m -= ((fast_sint_t)(s & 3) == (c0 >= 0));
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = i - 2;
+ m -= ((fast_sint_t)(s & 3) == (c1 >= 0));
+ }
+
+ for (; i >= 0; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = i + 1;
+ m -= ((fast_sint_t)(s & 3) == (c1 >= 0));
+ }
+
+ return n - 1 - m;
+}
+static void libsais_count_lms_suffixes_32s_2k(const sa_sint_t * RESTRICT T, sa_sint_t n, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets) {
+ const fast_sint_t prefetch_distance = 32;
+
+ memset(buckets, 0, 2 * (size_t)k * sizeof(sa_sint_t));
+
+ sa_sint_t i = n - 2;
+ fast_uint_t s = 1;
+ fast_sint_t c0 = T[n - 1];
+ fast_sint_t c1 = 0;
+
+ for (; i >= prefetch_distance + 3; i -= 4) {
+ prefetch(&T[i - 2 * prefetch_distance]);
+
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 0], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 1], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 2], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 3], 0)]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, (s & 3) == 1)]++;
+
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, (s & 3) == 1)]++;
+
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+ }
+
+ for (; i >= 0; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+ }
+
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, 0)]++;
+}
+static sa_sint_t libsais_count_and_gather_lms_suffixes_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t * RESTRICT buckets, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ memset(buckets, 0, 4 * ALPHABET_SIZE * sizeof(sa_sint_t));
+
+ fast_sint_t m = omp_block_start + omp_block_size - 1;
+
+ if (omp_block_size > 0) {
+ const fast_sint_t prefetch_distance = 128;
+
+ fast_sint_t i, j = m + 1, c0 = T[m], c1 = -1;
+
+ while (j < n && (c1 = T[j]) == c0) {
+ ++j;
+ }
+
+ fast_uint_t s = c0 >= c1;
+
+ for (i = m - 1, j = omp_block_start + 3; i >= j; i -= 4) {
+ prefetch(&T[i - prefetch_distance]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c0, s & 3)]++;
+
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 0);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c1, s & 3)]++;
+
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c0, s & 3)]++;
+
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 2);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c1, s & 3)]++;
+ }
+
+ for (j -= 3; i >= j; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c1, s & 3)]++;
+ }
+
+ c1 = (i >= 0) ? T[i] : -1;
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c0, s & 3)]++;
+ }
+
+ return (sa_sint_t)(omp_block_start + omp_block_size - 1 - m);
+}
+
+static sa_sint_t libsais_count_and_gather_lms_suffixes_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT buckets,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t m = 0;
+
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (n / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : n - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ m = libsais_count_and_gather_lms_suffixes_8u(T, SA, n, buckets, omp_block_start, omp_block_size);
+ }
+ }
+
+ return m;
+}
+
+static sa_sint_t libsais_count_and_gather_lms_suffixes_32s_4k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ memset(buckets, 0, 4 * (size_t)k * sizeof(sa_sint_t));
+
+ fast_sint_t m = omp_block_start + omp_block_size - 1;
+
+ if (omp_block_size > 0) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j = m + 1, c0 = T[m], c1 = -1;
+
+ while (j < n && (c1 = T[j]) == c0) {
+ ++j;
+ }
+
+ fast_uint_t s = c0 >= c1;
+
+ for (i = m - 1, j = omp_block_start + prefetch_distance + 3; i >= j; i -= 4) {
+ prefetch(&T[i - 2 * prefetch_distance]);
+
+ prefetchw(&buckets[BUCKETS_INDEX4(T[i - prefetch_distance - 0], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX4(T[i - prefetch_distance - 1], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX4(T[i - prefetch_distance - 2], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX4(T[i - prefetch_distance - 3], 0)]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c0, s & 3)]++;
+
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 0);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c1, s & 3)]++;
+
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c0, s & 3)]++;
+
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 2);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c1, s & 3)]++;
+ }
+
+ for (j -= prefetch_distance + 3; i >= j; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c1, s & 3)]++;
+ }
+
+ c1 = (i >= 0) ? T[i] : -1;
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX4((fast_uint_t)c0, s & 3)]++;
+ }
+
+ return (sa_sint_t)(omp_block_start + omp_block_size - 1 - m);
+}
+
+static sa_sint_t libsais_count_and_gather_lms_suffixes_32s_2k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ memset(buckets, 0, 2 * (size_t)k * sizeof(sa_sint_t));
+
+ fast_sint_t m = omp_block_start + omp_block_size - 1;
+
+ if (omp_block_size > 0) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j = m + 1, c0 = T[m], c1 = -1;
+
+ while (j < n && (c1 = T[j]) == c0) {
+ ++j;
+ }
+
+ fast_uint_t s = c0 >= c1;
+
+ for (i = m - 1, j = omp_block_start + prefetch_distance + 3; i >= j; i -= 4) {
+ prefetch(&T[i - 2 * prefetch_distance]);
+
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 0], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 1], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 2], 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 3], 0)]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, (s & 3) == 1)]++;
+
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 0);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, (s & 3) == 1)]++;
+
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 2);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+ }
+
+ for (j -= prefetch_distance + 3; i >= j; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+ }
+
+ c1 = (i >= 0) ? T[i] : -1;
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((s & 3) == 1);
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, (s & 3) == 1)]++;
+ }
+
+ return (sa_sint_t)(omp_block_start + omp_block_size - 1 - m);
+}
+
+static sa_sint_t libsais_count_and_gather_compacted_lms_suffixes_32s_2k(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ memset(buckets, 0, 2 * (size_t)k * sizeof(sa_sint_t));
+
+ fast_sint_t m = omp_block_start + omp_block_size - 1;
+
+ if (omp_block_size > 0) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j = m + 1, c0 = T[m], c1 = -1;
+
+ while (j < n && (c1 = T[j]) == c0) {
+ ++j;
+ }
+
+ fast_uint_t s = c0 >= c1;
+
+ for (i = m - 1, j = omp_block_start + prefetch_distance + 3; i >= j; i -= 4) {
+ prefetch(&T[i - 2 * prefetch_distance]);
+
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 0] & SAINT_MAX, 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 1] & SAINT_MAX, 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 2] & SAINT_MAX, 0)]);
+ prefetchw(&buckets[BUCKETS_INDEX2(T[i - prefetch_distance - 3] & SAINT_MAX, 0)]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((fast_sint_t)(s & 3) == (c0 >= 0));
+ c0 &= SAINT_MAX;
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, (s & 3) == 1)]++;
+
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 0);
+ m -= ((fast_sint_t)(s & 3) == (c1 >= 0));
+ c1 &= SAINT_MAX;
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 1);
+ m -= ((fast_sint_t)(s & 3) == (c0 >= 0));
+ c0 &= SAINT_MAX;
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, (s & 3) == 1)]++;
+
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i - 2);
+ m -= ((fast_sint_t)(s & 3) == (c1 >= 0));
+ c1 &= SAINT_MAX;
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+ }
+
+ for (j -= prefetch_distance + 3; i >= j; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((fast_sint_t)(s & 3) == (c1 >= 0));
+ c1 &= SAINT_MAX;
+ buckets[BUCKETS_INDEX2((fast_uint_t)c1, (s & 3) == 1)]++;
+ }
+
+ c1 = (i >= 0) ? T[i] : -1;
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ SA[m] = (sa_sint_t)(i + 1);
+ m -= ((fast_sint_t)(s & 3) == (c0 >= 0));
+ c0 &= SAINT_MAX;
+ buckets[BUCKETS_INDEX2((fast_uint_t)c0, (s & 3) == 1)]++;
+ }
+
+ return (sa_sint_t)(omp_block_start + omp_block_size - 1 - m);
+}
+static sa_sint_t libsais_count_and_gather_lms_suffixes_32s_4k_nofs_omp(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ sa_sint_t threads) {
+ sa_sint_t m = 0;
+ {
+ (void)(threads);
+
+ fast_sint_t omp_num_threads = 1;
+
+ if (omp_num_threads == 1) {
+ m = libsais_count_and_gather_lms_suffixes_32s_4k(T, SA, n, k, buckets, 0, n);
+ }
+ }
+
+ return m;
+}
+
+static sa_sint_t libsais_count_and_gather_lms_suffixes_32s_2k_nofs_omp(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ sa_sint_t threads) {
+ sa_sint_t m = 0;
+ {
+ (void)(threads);
+
+ fast_sint_t omp_num_threads = 1;
+
+ if (omp_num_threads == 1) {
+ m = libsais_count_and_gather_lms_suffixes_32s_2k(T, SA, n, k, buckets, 0, n);
+ }
+ }
+
+ return m;
+}
+
+static sa_sint_t libsais_count_and_gather_compacted_lms_suffixes_32s_2k_nofs_omp(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k,
+ sa_sint_t * RESTRICT buckets,
+ sa_sint_t threads) {
+ sa_sint_t m = 0;
+ {
+ (void)(threads);
+
+ fast_sint_t omp_num_threads = 1;
+
+ if (omp_num_threads == 1) {
+ m = libsais_count_and_gather_compacted_lms_suffixes_32s_2k(T, SA, n, k, buckets, 0, n);
+ }
+ }
+
+ return m;
+}
+
+static sa_sint_t libsais_count_and_gather_lms_suffixes_32s_4k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t m;
+ (void)(thread_state);
+
+ { m = libsais_count_and_gather_lms_suffixes_32s_4k_nofs_omp(T, SA, n, k, buckets, threads); }
+
+ return m;
+}
+
+static sa_sint_t libsais_count_and_gather_lms_suffixes_32s_2k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t m;
+ (void)(thread_state);
+
+ { m = libsais_count_and_gather_lms_suffixes_32s_2k_nofs_omp(T, SA, n, k, buckets, threads); }
+
+ return m;
+}
+
+static void libsais_count_and_gather_compacted_lms_suffixes_32s_2k_omp(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ (void)(thread_state);
+
+ { libsais_count_and_gather_compacted_lms_suffixes_32s_2k_nofs_omp(T, SA, n, k, buckets, threads); }
+}
+
+static void libsais_count_suffixes_32s(const sa_sint_t * RESTRICT T, sa_sint_t n, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets) {
+ const fast_sint_t prefetch_distance = 32;
+
+ memset(buckets, 0, (size_t)k * sizeof(sa_sint_t));
+
+ fast_sint_t i, j;
+ for (i = 0, j = (fast_sint_t)n - 7; i < j; i += 8) {
+ prefetch(&T[i + prefetch_distance]);
+
+ buckets[T[i + 0]]++;
+ buckets[T[i + 1]]++;
+ buckets[T[i + 2]]++;
+ buckets[T[i + 3]]++;
+ buckets[T[i + 4]]++;
+ buckets[T[i + 5]]++;
+ buckets[T[i + 6]]++;
+ buckets[T[i + 7]]++;
+ }
+
+ for (j += 7; i < j; i += 1) {
+ buckets[T[i]]++;
+ }
+}
+
+static void libsais_initialize_buckets_start_and_end_8u(sa_sint_t * RESTRICT buckets, sa_sint_t * RESTRICT freq) {
+ sa_sint_t * RESTRICT bucket_start = &buckets[6 * ALPHABET_SIZE];
+ sa_sint_t * RESTRICT bucket_end = &buckets[7 * ALPHABET_SIZE];
+
+ if (freq != NULL) {
+ fast_sint_t i, j;
+ sa_sint_t sum = 0;
+ for (i = BUCKETS_INDEX4(0, 0), j = 0; i <= BUCKETS_INDEX4(ALPHABET_SIZE - 1, 0);
+ i += BUCKETS_INDEX4(1, 0), j += 1) {
+ bucket_start[j] = sum;
+ sum += (freq[j] = buckets[i + BUCKETS_INDEX4(0, 0)] + buckets[i + BUCKETS_INDEX4(0, 1)] +
+ buckets[i + BUCKETS_INDEX4(0, 2)] + buckets[i + BUCKETS_INDEX4(0, 3)]);
+ bucket_end[j] = sum;
+ }
+ } else {
+ fast_sint_t i, j;
+ sa_sint_t sum = 0;
+ for (i = BUCKETS_INDEX4(0, 0), j = 0; i <= BUCKETS_INDEX4(ALPHABET_SIZE - 1, 0);
+ i += BUCKETS_INDEX4(1, 0), j += 1) {
+ bucket_start[j] = sum;
+ sum += buckets[i + BUCKETS_INDEX4(0, 0)] + buckets[i + BUCKETS_INDEX4(0, 1)] +
+ buckets[i + BUCKETS_INDEX4(0, 2)] + buckets[i + BUCKETS_INDEX4(0, 3)];
+ bucket_end[j] = sum;
+ }
+ }
+}
+
+static void libsais_initialize_buckets_start_and_end_32s_6k(sa_sint_t k, sa_sint_t * RESTRICT buckets) {
+ sa_sint_t * RESTRICT bucket_start = &buckets[4 * k];
+ sa_sint_t * RESTRICT bucket_end = &buckets[5 * k];
+
+ fast_sint_t i, j;
+ sa_sint_t sum = 0;
+ for (i = BUCKETS_INDEX4(0, 0), j = 0; i <= BUCKETS_INDEX4((fast_sint_t)k - 1, 0);
+ i += BUCKETS_INDEX4(1, 0), j += 1) {
+ bucket_start[j] = sum;
+ sum += buckets[i + BUCKETS_INDEX4(0, 0)] + buckets[i + BUCKETS_INDEX4(0, 1)] +
+ buckets[i + BUCKETS_INDEX4(0, 2)] + buckets[i + BUCKETS_INDEX4(0, 3)];
+ bucket_end[j] = sum;
+ }
+}
+
+static void libsais_initialize_buckets_start_and_end_32s_4k(sa_sint_t k, sa_sint_t * RESTRICT buckets) {
+ sa_sint_t * RESTRICT bucket_start = &buckets[2 * k];
+ sa_sint_t * RESTRICT bucket_end = &buckets[3 * k];
+
+ fast_sint_t i, j;
+ sa_sint_t sum = 0;
+ for (i = BUCKETS_INDEX2(0, 0), j = 0; i <= BUCKETS_INDEX2((fast_sint_t)k - 1, 0);
+ i += BUCKETS_INDEX2(1, 0), j += 1) {
+ bucket_start[j] = sum;
+ sum += buckets[i + BUCKETS_INDEX2(0, 0)] + buckets[i + BUCKETS_INDEX2(0, 1)];
+ bucket_end[j] = sum;
+ }
+}
+
+static void libsais_initialize_buckets_end_32s_2k(sa_sint_t k, sa_sint_t * RESTRICT buckets) {
+ fast_sint_t i;
+ sa_sint_t sum0 = 0;
+ for (i = BUCKETS_INDEX2(0, 0); i <= BUCKETS_INDEX2((fast_sint_t)k - 1, 0); i += BUCKETS_INDEX2(1, 0)) {
+ sum0 += buckets[i + BUCKETS_INDEX2(0, 0)] + buckets[i + BUCKETS_INDEX2(0, 1)];
+ buckets[i + BUCKETS_INDEX2(0, 0)] = sum0;
+ }
+}
+
+static void libsais_initialize_buckets_start_and_end_32s_2k(sa_sint_t k, sa_sint_t * RESTRICT buckets) {
+ fast_sint_t i, j;
+ for (i = BUCKETS_INDEX2(0, 0), j = 0; i <= BUCKETS_INDEX2((fast_sint_t)k - 1, 0);
+ i += BUCKETS_INDEX2(1, 0), j += 1) {
+ buckets[j] = buckets[i];
+ }
+
+ buckets[k] = 0;
+ memcpy(&buckets[k + 1], buckets, ((size_t)k - 1) * sizeof(sa_sint_t));
+}
+
+static void libsais_initialize_buckets_start_32s_1k(sa_sint_t k, sa_sint_t * RESTRICT buckets) {
+ fast_sint_t i;
+ sa_sint_t sum = 0;
+ for (i = 0; i <= (fast_sint_t)k - 1; i += 1) {
+ sa_sint_t tmp = buckets[i];
+ buckets[i] = sum;
+ sum += tmp;
+ }
+}
+
+static void libsais_initialize_buckets_end_32s_1k(sa_sint_t k, sa_sint_t * RESTRICT buckets) {
+ fast_sint_t i;
+ sa_sint_t sum = 0;
+ for (i = 0; i <= (fast_sint_t)k - 1; i += 1) {
+ sum += buckets[i];
+ buckets[i] = sum;
+ }
+}
+
+static sa_sint_t libsais_initialize_buckets_for_lms_suffixes_radix_sort_8u(const u8 * RESTRICT T,
+ sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix) {
+ {
+ fast_uint_t s = 0;
+ fast_sint_t c0 = T[first_lms_suffix];
+ fast_sint_t c1 = 0;
+
+ for (; --first_lms_suffix >= 0;) {
+ c1 = c0;
+ c0 = T[first_lms_suffix];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ buckets[BUCKETS_INDEX4((fast_uint_t)c1, s & 3)]--;
+ }
+
+ buckets[BUCKETS_INDEX4((fast_uint_t)c0, (s << 1) & 3)]--;
+ }
+
+ {
+ sa_sint_t * RESTRICT temp_bucket = &buckets[4 * ALPHABET_SIZE];
+
+ fast_sint_t i, j;
+ sa_sint_t sum = 0;
+ for (i = BUCKETS_INDEX4(0, 0), j = BUCKETS_INDEX2(0, 0); i <= BUCKETS_INDEX4(ALPHABET_SIZE - 1, 0);
+ i += BUCKETS_INDEX4(1, 0), j += BUCKETS_INDEX2(1, 0)) {
+ temp_bucket[j + BUCKETS_INDEX2(0, 1)] = sum;
+ sum += buckets[i + BUCKETS_INDEX4(0, 1)] + buckets[i + BUCKETS_INDEX4(0, 3)];
+ temp_bucket[j] = sum;
+ }
+
+ return sum;
+ }
+}
+
+static void libsais_initialize_buckets_for_lms_suffixes_radix_sort_32s_2k(const sa_sint_t * RESTRICT T, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix) {
+ buckets[BUCKETS_INDEX2(T[first_lms_suffix], 0)]++;
+ buckets[BUCKETS_INDEX2(T[first_lms_suffix], 1)]--;
+
+ fast_sint_t i;
+ sa_sint_t sum0 = 0, sum1 = 0;
+ for (i = BUCKETS_INDEX2(0, 0); i <= BUCKETS_INDEX2((fast_sint_t)k - 1, 0); i += BUCKETS_INDEX2(1, 0)) {
+ sum0 += buckets[i + BUCKETS_INDEX2(0, 0)] + buckets[i + BUCKETS_INDEX2(0, 1)];
+ sum1 += buckets[i + BUCKETS_INDEX2(0, 1)];
+
+ buckets[i + BUCKETS_INDEX2(0, 0)] = sum0;
+ buckets[i + BUCKETS_INDEX2(0, 1)] = sum1;
+ }
+}
+
+static sa_sint_t libsais_initialize_buckets_for_lms_suffixes_radix_sort_32s_6k(const sa_sint_t * RESTRICT T,
+ sa_sint_t k,
+ sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix) {
+ {
+ fast_uint_t s = 0;
+ fast_sint_t c0 = T[first_lms_suffix];
+ fast_sint_t c1 = 0;
+
+ for (; --first_lms_suffix >= 0;) {
+ c1 = c0;
+ c0 = T[first_lms_suffix];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ buckets[BUCKETS_INDEX4((fast_uint_t)c1, s & 3)]--;
+ }
+
+ buckets[BUCKETS_INDEX4((fast_uint_t)c0, (s << 1) & 3)]--;
+ }
+
+ {
+ sa_sint_t * RESTRICT temp_bucket = &buckets[4 * k];
+
+ fast_sint_t i, j;
+ sa_sint_t sum = 0;
+ for (i = BUCKETS_INDEX4(0, 0), j = 0; i <= BUCKETS_INDEX4((fast_sint_t)k - 1, 0);
+ i += BUCKETS_INDEX4(1, 0), j += 1) {
+ sum += buckets[i + BUCKETS_INDEX4(0, 1)] + buckets[i + BUCKETS_INDEX4(0, 3)];
+ temp_bucket[j] = sum;
+ }
+
+ return sum;
+ }
+}
+
+static void libsais_initialize_buckets_for_radix_and_partial_sorting_32s_4k(const sa_sint_t * RESTRICT T, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix) {
+ sa_sint_t * RESTRICT bucket_start = &buckets[2 * k];
+ sa_sint_t * RESTRICT bucket_end = &buckets[3 * k];
+
+ buckets[BUCKETS_INDEX2(T[first_lms_suffix], 0)]++;
+ buckets[BUCKETS_INDEX2(T[first_lms_suffix], 1)]--;
+
+ fast_sint_t i, j;
+ sa_sint_t sum0 = 0, sum1 = 0;
+ for (i = BUCKETS_INDEX2(0, 0), j = 0; i <= BUCKETS_INDEX2((fast_sint_t)k - 1, 0);
+ i += BUCKETS_INDEX2(1, 0), j += 1) {
+ bucket_start[j] = sum1;
+
+ sum0 += buckets[i + BUCKETS_INDEX2(0, 1)];
+ sum1 += buckets[i + BUCKETS_INDEX2(0, 0)] + buckets[i + BUCKETS_INDEX2(0, 1)];
+ buckets[i + BUCKETS_INDEX2(0, 1)] = sum0;
+
+ bucket_end[j] = sum1;
+ }
+}
+
+static void libsais_radix_sort_lms_suffixes_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + prefetch_distance + 3; i >= j; i -= 4) {
+ prefetch(&SA[i - 2 * prefetch_distance]);
+
+ prefetch(&T[SA[i - prefetch_distance - 0]]);
+ prefetch(&T[SA[i - prefetch_distance - 1]]);
+ prefetch(&T[SA[i - prefetch_distance - 2]]);
+ prefetch(&T[SA[i - prefetch_distance - 3]]);
+
+ sa_sint_t p0 = SA[i - 0];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p0], 0)]] = p0;
+ sa_sint_t p1 = SA[i - 1];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p1], 0)]] = p1;
+ sa_sint_t p2 = SA[i - 2];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p2], 0)]] = p2;
+ sa_sint_t p3 = SA[i - 3];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p3], 0)]] = p3;
+ }
+
+ for (j -= prefetch_distance + 3; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p], 0)]] = p;
+ }
+}
+
+static void libsais_radix_sort_lms_suffixes_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t m, sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_num_threads = 1;
+
+ if (omp_num_threads == 1) {
+ libsais_radix_sort_lms_suffixes_8u(T, SA, &buckets[4 * ALPHABET_SIZE], (fast_sint_t)n - (fast_sint_t)m + 1,
+ (fast_sint_t)m - 1);
+ }
+ }
+}
+
+static void libsais_radix_sort_lms_suffixes_32s_6k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + 2 * prefetch_distance + 3; i >= j; i -= 4) {
+ prefetch(&SA[i - 3 * prefetch_distance]);
+
+ prefetch(&T[SA[i - 2 * prefetch_distance - 0]]);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 1]]);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 2]]);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 3]]);
+
+ prefetchw(&induction_bucket[T[SA[i - prefetch_distance - 0]]]);
+ prefetchw(&induction_bucket[T[SA[i - prefetch_distance - 1]]]);
+ prefetchw(&induction_bucket[T[SA[i - prefetch_distance - 2]]]);
+ prefetchw(&induction_bucket[T[SA[i - prefetch_distance - 3]]]);
+
+ sa_sint_t p0 = SA[i - 0];
+ SA[--induction_bucket[T[p0]]] = p0;
+ sa_sint_t p1 = SA[i - 1];
+ SA[--induction_bucket[T[p1]]] = p1;
+ sa_sint_t p2 = SA[i - 2];
+ SA[--induction_bucket[T[p2]]] = p2;
+ sa_sint_t p3 = SA[i - 3];
+ SA[--induction_bucket[T[p3]]] = p3;
+ }
+
+ for (j -= 2 * prefetch_distance + 3; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ SA[--induction_bucket[T[p]]] = p;
+ }
+}
+
+static void libsais_radix_sort_lms_suffixes_32s_2k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + 2 * prefetch_distance + 3; i >= j; i -= 4) {
+ prefetch(&SA[i - 3 * prefetch_distance]);
+
+ prefetch(&T[SA[i - 2 * prefetch_distance - 0]]);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 1]]);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 2]]);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 3]]);
+
+ prefetchw(&induction_bucket[BUCKETS_INDEX2(T[SA[i - prefetch_distance - 0]], 0)]);
+ prefetchw(&induction_bucket[BUCKETS_INDEX2(T[SA[i - prefetch_distance - 1]], 0)]);
+ prefetchw(&induction_bucket[BUCKETS_INDEX2(T[SA[i - prefetch_distance - 2]], 0)]);
+ prefetchw(&induction_bucket[BUCKETS_INDEX2(T[SA[i - prefetch_distance - 3]], 0)]);
+
+ sa_sint_t p0 = SA[i - 0];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p0], 0)]] = p0;
+ sa_sint_t p1 = SA[i - 1];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p1], 0)]] = p1;
+ sa_sint_t p2 = SA[i - 2];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p2], 0)]] = p2;
+ sa_sint_t p3 = SA[i - 3];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p3], 0)]] = p3;
+ }
+
+ for (j -= 2 * prefetch_distance + 3; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ SA[--induction_bucket[BUCKETS_INDEX2(T[p], 0)]] = p;
+ }
+}
+static void libsais_radix_sort_lms_suffixes_32s_6k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t m, sa_sint_t * RESTRICT induction_bucket,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (threads == 1 || m < 65536) {
+ libsais_radix_sort_lms_suffixes_32s_6k(T, SA, induction_bucket, (fast_sint_t)n - (fast_sint_t)m + 1,
+ (fast_sint_t)m - 1);
+ }
+ (void)(thread_state);
+}
+
+static void libsais_radix_sort_lms_suffixes_32s_2k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t m, sa_sint_t * RESTRICT induction_bucket,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (threads == 1 || m < 65536) {
+ libsais_radix_sort_lms_suffixes_32s_2k(T, SA, induction_bucket, (fast_sint_t)n - (fast_sint_t)m + 1,
+ (fast_sint_t)m - 1);
+ }
+ (void)(thread_state);
+}
+
+static sa_sint_t libsais_radix_sort_lms_suffixes_32s_1k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT buckets) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t i = n - 2;
+ sa_sint_t m = 0;
+ fast_uint_t s = 1;
+ fast_sint_t c0 = T[n - 1];
+ fast_sint_t c1 = 0;
+ fast_sint_t c2 = 0;
+
+ for (; i >= prefetch_distance + 3; i -= 4) {
+ prefetch(&T[i - 2 * prefetch_distance]);
+
+ prefetchw(&buckets[T[i - prefetch_distance - 0]]);
+ prefetchw(&buckets[T[i - prefetch_distance - 1]]);
+ prefetchw(&buckets[T[i - prefetch_distance - 2]]);
+ prefetchw(&buckets[T[i - prefetch_distance - 3]]);
+
+ c1 = T[i - 0];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ if ((s & 3) == 1) {
+ SA[--buckets[c2 = c0]] = i + 1;
+ m++;
+ }
+
+ c0 = T[i - 1];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ if ((s & 3) == 1) {
+ SA[--buckets[c2 = c1]] = i - 0;
+ m++;
+ }
+
+ c1 = T[i - 2];
+ s = (s << 1) + (fast_uint_t)(c1 > (c0 - (fast_sint_t)(s & 1)));
+ if ((s & 3) == 1) {
+ SA[--buckets[c2 = c0]] = i - 1;
+ m++;
+ }
+
+ c0 = T[i - 3];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ if ((s & 3) == 1) {
+ SA[--buckets[c2 = c1]] = i - 2;
+ m++;
+ }
+ }
+
+ for (; i >= 0; i -= 1) {
+ c1 = c0;
+ c0 = T[i];
+ s = (s << 1) + (fast_uint_t)(c0 > (c1 - (fast_sint_t)(s & 1)));
+ if ((s & 3) == 1) {
+ SA[--buckets[c2 = c1]] = i + 1;
+ m++;
+ }
+ }
+
+ if (m > 1) {
+ SA[buckets[c2]] = 0;
+ }
+
+ return m;
+}
+
+static void libsais_radix_sort_set_markers_32s_6k(sa_sint_t * RESTRICT SA, sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 3; i < j; i += 4) {
+ prefetch(&induction_bucket[i + 2 * prefetch_distance]);
+
+ prefetchw(&SA[induction_bucket[i + prefetch_distance + 0]]);
+ prefetchw(&SA[induction_bucket[i + prefetch_distance + 1]]);
+ prefetchw(&SA[induction_bucket[i + prefetch_distance + 2]]);
+ prefetchw(&SA[induction_bucket[i + prefetch_distance + 3]]);
+
+ SA[induction_bucket[i + 0]] |= SAINT_MIN;
+ SA[induction_bucket[i + 1]] |= SAINT_MIN;
+ SA[induction_bucket[i + 2]] |= SAINT_MIN;
+ SA[induction_bucket[i + 3]] |= SAINT_MIN;
+ }
+
+ for (j += prefetch_distance + 3; i < j; i += 1) {
+ SA[induction_bucket[i]] |= SAINT_MIN;
+ }
+}
+
+static void libsais_radix_sort_set_markers_32s_4k(sa_sint_t * RESTRICT SA, sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 3; i < j; i += 4) {
+ prefetch(&induction_bucket[BUCKETS_INDEX2(i + 2 * prefetch_distance, 0)]);
+
+ prefetchw(&SA[induction_bucket[BUCKETS_INDEX2(i + prefetch_distance + 0, 0)]]);
+ prefetchw(&SA[induction_bucket[BUCKETS_INDEX2(i + prefetch_distance + 1, 0)]]);
+ prefetchw(&SA[induction_bucket[BUCKETS_INDEX2(i + prefetch_distance + 2, 0)]]);
+ prefetchw(&SA[induction_bucket[BUCKETS_INDEX2(i + prefetch_distance + 3, 0)]]);
+
+ SA[induction_bucket[BUCKETS_INDEX2(i + 0, 0)]] |= SUFFIX_GROUP_MARKER;
+ SA[induction_bucket[BUCKETS_INDEX2(i + 1, 0)]] |= SUFFIX_GROUP_MARKER;
+ SA[induction_bucket[BUCKETS_INDEX2(i + 2, 0)]] |= SUFFIX_GROUP_MARKER;
+ SA[induction_bucket[BUCKETS_INDEX2(i + 3, 0)]] |= SUFFIX_GROUP_MARKER;
+ }
+
+ for (j += prefetch_distance + 3; i < j; i += 1) {
+ SA[induction_bucket[BUCKETS_INDEX2(i, 0)]] |= SUFFIX_GROUP_MARKER;
+ }
+}
+
+static void libsais_radix_sort_set_markers_32s_6k_omp(sa_sint_t * RESTRICT SA, sa_sint_t k,
+ sa_sint_t * RESTRICT induction_bucket, sa_sint_t threads) {
+ {
+ (void)(threads);
+
+ fast_sint_t omp_block_start = 0;
+ fast_sint_t omp_block_size = (fast_sint_t)k - 1;
+ libsais_radix_sort_set_markers_32s_6k(SA, induction_bucket, omp_block_start, omp_block_size);
+ }
+}
+
+static void libsais_radix_sort_set_markers_32s_4k_omp(sa_sint_t * RESTRICT SA, sa_sint_t k,
+ sa_sint_t * RESTRICT induction_bucket, sa_sint_t threads) {
+ {
+ (void)(threads);
+
+ fast_sint_t omp_block_start = 0;
+ fast_sint_t omp_block_size = (fast_sint_t)k - 1;
+ libsais_radix_sort_set_markers_32s_4k(SA, induction_bucket, omp_block_start, omp_block_size);
+ }
+}
+
+static void libsais_initialize_buckets_for_partial_sorting_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix,
+ sa_sint_t left_suffixes_count) {
+ sa_sint_t * RESTRICT temp_bucket = &buckets[4 * ALPHABET_SIZE];
+
+ buckets[BUCKETS_INDEX4((fast_uint_t)T[first_lms_suffix], 1)]++;
+
+ fast_sint_t i, j;
+ sa_sint_t sum0 = left_suffixes_count + 1, sum1 = 0;
+ for (i = BUCKETS_INDEX4(0, 0), j = BUCKETS_INDEX2(0, 0); i <= BUCKETS_INDEX4(ALPHABET_SIZE - 1, 0);
+ i += BUCKETS_INDEX4(1, 0), j += BUCKETS_INDEX2(1, 0)) {
+ temp_bucket[j + BUCKETS_INDEX2(0, 0)] = sum0;
+
+ sum0 += buckets[i + BUCKETS_INDEX4(0, 0)] + buckets[i + BUCKETS_INDEX4(0, 2)];
+ sum1 += buckets[i + BUCKETS_INDEX4(0, 1)];
+
+ buckets[j + BUCKETS_INDEX2(0, 0)] = sum0;
+ buckets[j + BUCKETS_INDEX2(0, 1)] = sum1;
+ }
+}
+
+static void libsais_initialize_buckets_for_partial_sorting_32s_6k(const sa_sint_t * RESTRICT T, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix,
+ sa_sint_t left_suffixes_count) {
+ sa_sint_t * RESTRICT temp_bucket = &buckets[4 * k];
+
+ fast_sint_t i, j;
+ sa_sint_t sum0 = left_suffixes_count + 1, sum1 = 0, sum2 = 0;
+ for (first_lms_suffix = T[first_lms_suffix], i = BUCKETS_INDEX4(0, 0), j = BUCKETS_INDEX2(0, 0);
+ i <= BUCKETS_INDEX4((fast_sint_t)first_lms_suffix - 1, 0);
+ i += BUCKETS_INDEX4(1, 0), j += BUCKETS_INDEX2(1, 0)) {
+ sa_sint_t SS = buckets[i + BUCKETS_INDEX4(0, 0)];
+ sa_sint_t LS = buckets[i + BUCKETS_INDEX4(0, 1)];
+ sa_sint_t SL = buckets[i + BUCKETS_INDEX4(0, 2)];
+ sa_sint_t LL = buckets[i + BUCKETS_INDEX4(0, 3)];
+
+ buckets[i + BUCKETS_INDEX4(0, 0)] = sum0;
+ buckets[i + BUCKETS_INDEX4(0, 1)] = sum2;
+ buckets[i + BUCKETS_INDEX4(0, 2)] = 0;
+ buckets[i + BUCKETS_INDEX4(0, 3)] = 0;
+
+ sum0 += SS + SL;
+ sum1 += LS;
+ sum2 += LS + LL;
+
+ temp_bucket[j + BUCKETS_INDEX2(0, 0)] = sum0;
+ temp_bucket[j + BUCKETS_INDEX2(0, 1)] = sum1;
+ }
+
+ for (sum1 += 1; i <= BUCKETS_INDEX4((fast_sint_t)k - 1, 0); i += BUCKETS_INDEX4(1, 0), j += BUCKETS_INDEX2(1, 0)) {
+ sa_sint_t SS = buckets[i + BUCKETS_INDEX4(0, 0)];
+ sa_sint_t LS = buckets[i + BUCKETS_INDEX4(0, 1)];
+ sa_sint_t SL = buckets[i + BUCKETS_INDEX4(0, 2)];
+ sa_sint_t LL = buckets[i + BUCKETS_INDEX4(0, 3)];
+
+ buckets[i + BUCKETS_INDEX4(0, 0)] = sum0;
+ buckets[i + BUCKETS_INDEX4(0, 1)] = sum2;
+ buckets[i + BUCKETS_INDEX4(0, 2)] = 0;
+ buckets[i + BUCKETS_INDEX4(0, 3)] = 0;
+
+ sum0 += SS + SL;
+ sum1 += LS;
+ sum2 += LS + LL;
+
+ temp_bucket[j + BUCKETS_INDEX2(0, 0)] = sum0;
+ temp_bucket[j + BUCKETS_INDEX2(0, 1)] = sum1;
+ }
+}
+
+static sa_sint_t libsais_partial_sorting_scan_left_to_right_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT buckets, sa_sint_t d,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT induction_bucket = &buckets[4 * ALPHABET_SIZE];
+ sa_sint_t * RESTRICT distinct_names = &buckets[2 * ALPHABET_SIZE];
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 1; i < j; i += 2) {
+ prefetch(&SA[i + 2 * prefetch_distance]);
+
+ prefetch(&T[SA[i + prefetch_distance + 0] & SAINT_MAX] - 1);
+ prefetch(&T[SA[i + prefetch_distance + 0] & SAINT_MAX] - 2);
+ prefetch(&T[SA[i + prefetch_distance + 1] & SAINT_MAX] - 1);
+ prefetch(&T[SA[i + prefetch_distance + 1] & SAINT_MAX] - 2);
+
+ sa_sint_t p0 = SA[i + 0];
+ d += (p0 < 0);
+ p0 &= SAINT_MAX;
+ sa_sint_t v0 = BUCKETS_INDEX2(T[p0 - 1], T[p0 - 2] >= T[p0 - 1]);
+ SA[induction_bucket[v0]++] = (p0 - 1) | ((sa_sint_t)(distinct_names[v0] != d) << (SAINT_BIT - 1));
+ distinct_names[v0] = d;
+
+ sa_sint_t p1 = SA[i + 1];
+ d += (p1 < 0);
+ p1 &= SAINT_MAX;
+ sa_sint_t v1 = BUCKETS_INDEX2(T[p1 - 1], T[p1 - 2] >= T[p1 - 1]);
+ SA[induction_bucket[v1]++] = (p1 - 1) | ((sa_sint_t)(distinct_names[v1] != d) << (SAINT_BIT - 1));
+ distinct_names[v1] = d;
+ }
+
+ for (j += prefetch_distance + 1; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ d += (p < 0);
+ p &= SAINT_MAX;
+ sa_sint_t v = BUCKETS_INDEX2(T[p - 1], T[p - 2] >= T[p - 1]);
+ SA[induction_bucket[v]++] = (p - 1) | ((sa_sint_t)(distinct_names[v] != d) << (SAINT_BIT - 1));
+ distinct_names[v] = d;
+ }
+
+ return d;
+}
+static sa_sint_t libsais_partial_sorting_scan_left_to_right_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT buckets,
+ sa_sint_t left_suffixes_count, sa_sint_t d,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t * RESTRICT induction_bucket = &buckets[4 * ALPHABET_SIZE];
+ sa_sint_t * RESTRICT distinct_names = &buckets[2 * ALPHABET_SIZE];
+
+ SA[induction_bucket[BUCKETS_INDEX2(T[n - 1], T[n - 2] >= T[n - 1])]++] = (n - 1) | SAINT_MIN;
+ distinct_names[BUCKETS_INDEX2(T[n - 1], T[n - 2] >= T[n - 1])] = ++d;
+
+ if (threads == 1 || left_suffixes_count < 65536) {
+ d = libsais_partial_sorting_scan_left_to_right_8u(T, SA, buckets, d, 0, left_suffixes_count);
+ }
+ (void)(thread_state);
+ return d;
+}
+
+static sa_sint_t libsais_partial_sorting_scan_left_to_right_32s_6k(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT buckets, sa_sint_t d,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - 2 * prefetch_distance - 1; i < j; i += 2) {
+ prefetch(&SA[i + 3 * prefetch_distance]);
+
+ prefetch(&T[SA[i + 2 * prefetch_distance + 0] & SAINT_MAX] - 1);
+ prefetch(&T[SA[i + 2 * prefetch_distance + 0] & SAINT_MAX] - 2);
+ prefetch(&T[SA[i + 2 * prefetch_distance + 1] & SAINT_MAX] - 1);
+ prefetch(&T[SA[i + 2 * prefetch_distance + 1] & SAINT_MAX] - 2);
+
+ sa_sint_t p0 = SA[i + prefetch_distance + 0] & SAINT_MAX;
+ sa_sint_t v0 = BUCKETS_INDEX4(T[p0 - (p0 > 0)], 0);
+ prefetchw(&buckets[v0]);
+ sa_sint_t p1 = SA[i + prefetch_distance + 1] & SAINT_MAX;
+ sa_sint_t v1 = BUCKETS_INDEX4(T[p1 - (p1 > 0)], 0);
+ prefetchw(&buckets[v1]);
+
+ sa_sint_t p2 = SA[i + 0];
+ d += (p2 < 0);
+ p2 &= SAINT_MAX;
+ sa_sint_t v2 = BUCKETS_INDEX4(T[p2 - 1], T[p2 - 2] >= T[p2 - 1]);
+ SA[buckets[v2]++] = (p2 - 1) | ((sa_sint_t)(buckets[2 + v2] != d) << (SAINT_BIT - 1));
+ buckets[2 + v2] = d;
+
+ sa_sint_t p3 = SA[i + 1];
+ d += (p3 < 0);
+ p3 &= SAINT_MAX;
+ sa_sint_t v3 = BUCKETS_INDEX4(T[p3 - 1], T[p3 - 2] >= T[p3 - 1]);
+ SA[buckets[v3]++] = (p3 - 1) | ((sa_sint_t)(buckets[2 + v3] != d) << (SAINT_BIT - 1));
+ buckets[2 + v3] = d;
+ }
+
+ for (j += 2 * prefetch_distance + 1; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ d += (p < 0);
+ p &= SAINT_MAX;
+ sa_sint_t v = BUCKETS_INDEX4(T[p - 1], T[p - 2] >= T[p - 1]);
+ SA[buckets[v]++] = (p - 1) | ((sa_sint_t)(buckets[2 + v] != d) << (SAINT_BIT - 1));
+ buckets[2 + v] = d;
+ }
+
+ return d;
+}
+
+static sa_sint_t libsais_partial_sorting_scan_left_to_right_32s_4k(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets, sa_sint_t d,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT induction_bucket = &buckets[2 * k];
+ sa_sint_t * RESTRICT distinct_names = &buckets[0 * k];
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - 2 * prefetch_distance - 1; i < j; i += 2) {
+ prefetchw(&SA[i + 3 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i + 2 * prefetch_distance + 0];
+ const sa_sint_t * Ts0 = &T[s0 & ~SUFFIX_GROUP_MARKER] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ Ts0--;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i + 2 * prefetch_distance + 1];
+ const sa_sint_t * Ts1 = &T[s1 & ~SUFFIX_GROUP_MARKER] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ Ts1--;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ sa_sint_t s2 = SA[i + 1 * prefetch_distance + 0];
+ if (s2 > 0) {
+ const fast_sint_t Ts2 = T[(s2 & ~SUFFIX_GROUP_MARKER) - 1];
+ prefetchw(&induction_bucket[Ts2]);
+ prefetchw(&distinct_names[BUCKETS_INDEX2(Ts2, 0)]);
+ }
+ sa_sint_t s3 = SA[i + 1 * prefetch_distance + 1];
+ if (s3 > 0) {
+ const fast_sint_t Ts3 = T[(s3 & ~SUFFIX_GROUP_MARKER) - 1];
+ prefetchw(&induction_bucket[Ts3]);
+ prefetchw(&distinct_names[BUCKETS_INDEX2(Ts3, 0)]);
+ }
+
+ sa_sint_t p0 = SA[i + 0];
+ SA[i + 0] = p0 & SAINT_MAX;
+ if (p0 > 0) {
+ SA[i + 0] = 0;
+ d += (p0 >> (SUFFIX_GROUP_BIT - 1));
+ p0 &= ~SUFFIX_GROUP_MARKER;
+ sa_sint_t v0 = BUCKETS_INDEX2(T[p0 - 1], T[p0 - 2] < T[p0 - 1]);
+ SA[induction_bucket[T[p0 - 1]]++] = (p0 - 1) | ((sa_sint_t)(T[p0 - 2] < T[p0 - 1]) << (SAINT_BIT - 1)) |
+ ((sa_sint_t)(distinct_names[v0] != d) << (SUFFIX_GROUP_BIT - 1));
+ distinct_names[v0] = d;
+ }
+
+ sa_sint_t p1 = SA[i + 1];
+ SA[i + 1] = p1 & SAINT_MAX;
+ if (p1 > 0) {
+ SA[i + 1] = 0;
+ d += (p1 >> (SUFFIX_GROUP_BIT - 1));
+ p1 &= ~SUFFIX_GROUP_MARKER;
+ sa_sint_t v1 = BUCKETS_INDEX2(T[p1 - 1], T[p1 - 2] < T[p1 - 1]);
+ SA[induction_bucket[T[p1 - 1]]++] = (p1 - 1) | ((sa_sint_t)(T[p1 - 2] < T[p1 - 1]) << (SAINT_BIT - 1)) |
+ ((sa_sint_t)(distinct_names[v1] != d) << (SUFFIX_GROUP_BIT - 1));
+ distinct_names[v1] = d;
+ }
+ }
+
+ for (j += 2 * prefetch_distance + 1; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p & SAINT_MAX;
+ if (p > 0) {
+ SA[i] = 0;
+ d += (p >> (SUFFIX_GROUP_BIT - 1));
+ p &= ~SUFFIX_GROUP_MARKER;
+ sa_sint_t v = BUCKETS_INDEX2(T[p - 1], T[p - 2] < T[p - 1]);
+ SA[induction_bucket[T[p - 1]]++] = (p - 1) | ((sa_sint_t)(T[p - 2] < T[p - 1]) << (SAINT_BIT - 1)) |
+ ((sa_sint_t)(distinct_names[v] != d) << (SUFFIX_GROUP_BIT - 1));
+ distinct_names[v] = d;
+ }
+ }
+
+ return d;
+}
+
+static void libsais_partial_sorting_scan_left_to_right_32s_1k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - 2 * prefetch_distance - 1; i < j; i += 2) {
+ prefetchw(&SA[i + 3 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i + 2 * prefetch_distance + 0];
+ const sa_sint_t * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i + 2 * prefetch_distance + 1];
+ const sa_sint_t * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ sa_sint_t s2 = SA[i + 1 * prefetch_distance + 0];
+ if (s2 > 0) {
+ prefetchw(&induction_bucket[T[s2 - 1]]);
+ prefetch(&T[s2] - 2);
+ }
+ sa_sint_t s3 = SA[i + 1 * prefetch_distance + 1];
+ if (s3 > 0) {
+ prefetchw(&induction_bucket[T[s3 - 1]]);
+ prefetch(&T[s3] - 2);
+ }
+
+ sa_sint_t p0 = SA[i + 0];
+ SA[i + 0] = p0 & SAINT_MAX;
+ if (p0 > 0) {
+ SA[i + 0] = 0;
+ SA[induction_bucket[T[p0 - 1]]++] = (p0 - 1) | ((sa_sint_t)(T[p0 - 2] < T[p0 - 1]) << (SAINT_BIT - 1));
+ }
+ sa_sint_t p1 = SA[i + 1];
+ SA[i + 1] = p1 & SAINT_MAX;
+ if (p1 > 0) {
+ SA[i + 1] = 0;
+ SA[induction_bucket[T[p1 - 1]]++] = (p1 - 1) | ((sa_sint_t)(T[p1 - 2] < T[p1 - 1]) << (SAINT_BIT - 1));
+ }
+ }
+
+ for (j += 2 * prefetch_distance + 1; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p & SAINT_MAX;
+ if (p > 0) {
+ SA[i] = 0;
+ SA[induction_bucket[T[p - 1]]++] = (p - 1) | ((sa_sint_t)(T[p - 2] < T[p - 1]) << (SAINT_BIT - 1));
+ }
+ }
+}
+static sa_sint_t libsais_partial_sorting_scan_left_to_right_32s_6k_omp(
+ const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t * RESTRICT buckets,
+ sa_sint_t left_suffixes_count, sa_sint_t d, sa_sint_t threads, LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ SA[buckets[BUCKETS_INDEX4(T[n - 1], T[n - 2] >= T[n - 1])]++] = (n - 1) | SAINT_MIN;
+ buckets[2 + BUCKETS_INDEX4(T[n - 1], T[n - 2] >= T[n - 1])] = ++d;
+
+ if (threads == 1 || left_suffixes_count < 65536) {
+ d = libsais_partial_sorting_scan_left_to_right_32s_6k(T, SA, buckets, d, 0, left_suffixes_count);
+ }
+ (void)(thread_state);
+ return d;
+}
+
+static sa_sint_t libsais_partial_sorting_scan_left_to_right_32s_4k_omp(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ sa_sint_t d, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t * RESTRICT induction_bucket = &buckets[2 * k];
+ sa_sint_t * RESTRICT distinct_names = &buckets[0 * k];
+
+ SA[induction_bucket[T[n - 1]]++] =
+ (n - 1) | ((sa_sint_t)(T[n - 2] < T[n - 1]) << (SAINT_BIT - 1)) | SUFFIX_GROUP_MARKER;
+ distinct_names[BUCKETS_INDEX2(T[n - 1], T[n - 2] < T[n - 1])] = ++d;
+
+ if (threads == 1 || n < 65536) {
+ d = libsais_partial_sorting_scan_left_to_right_32s_4k(T, SA, k, buckets, d, 0, n);
+ }
+ (void)(thread_state);
+ return d;
+}
+
+static void libsais_partial_sorting_scan_left_to_right_32s_1k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT buckets,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ SA[buckets[T[n - 1]]++] = (n - 1) | ((sa_sint_t)(T[n - 2] < T[n - 1]) << (SAINT_BIT - 1));
+
+ if (threads == 1 || n < 65536) {
+ libsais_partial_sorting_scan_left_to_right_32s_1k(T, SA, buckets, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static void libsais_partial_sorting_shift_markers_8u_omp(sa_sint_t * RESTRICT SA, sa_sint_t n,
+ const sa_sint_t * RESTRICT buckets, sa_sint_t threads) {
+ const fast_sint_t prefetch_distance = 32;
+
+ const sa_sint_t * RESTRICT temp_bucket = &buckets[4 * ALPHABET_SIZE];
+
+ fast_sint_t c;
+ (void)(threads);
+ (void)(n);
+
+ for (c = BUCKETS_INDEX2(ALPHABET_SIZE - 1, 0); c >= BUCKETS_INDEX2(1, 0); c -= BUCKETS_INDEX2(1, 0)) {
+ fast_sint_t i, j;
+ sa_sint_t s = SAINT_MIN;
+ for (i = (fast_sint_t)temp_bucket[c] - 1, j = (fast_sint_t)buckets[c - BUCKETS_INDEX2(1, 0)] + 3; i >= j;
+ i -= 4) {
+ prefetchw(&SA[i - prefetch_distance]);
+
+ sa_sint_t p0 = SA[i - 0], q0 = (p0 & SAINT_MIN) ^ s;
+ s = s ^ q0;
+ SA[i - 0] = p0 ^ q0;
+ sa_sint_t p1 = SA[i - 1], q1 = (p1 & SAINT_MIN) ^ s;
+ s = s ^ q1;
+ SA[i - 1] = p1 ^ q1;
+ sa_sint_t p2 = SA[i - 2], q2 = (p2 & SAINT_MIN) ^ s;
+ s = s ^ q2;
+ SA[i - 2] = p2 ^ q2;
+ sa_sint_t p3 = SA[i - 3], q3 = (p3 & SAINT_MIN) ^ s;
+ s = s ^ q3;
+ SA[i - 3] = p3 ^ q3;
+ }
+
+ for (j -= 3; i >= j; i -= 1) {
+ sa_sint_t p = SA[i], q = (p & SAINT_MIN) ^ s;
+ s = s ^ q;
+ SA[i] = p ^ q;
+ }
+ }
+}
+
+static void libsais_partial_sorting_shift_markers_32s_6k_omp(sa_sint_t * RESTRICT SA, sa_sint_t k,
+ const sa_sint_t * RESTRICT buckets, sa_sint_t threads) {
+ const fast_sint_t prefetch_distance = 32;
+
+ const sa_sint_t * RESTRICT temp_bucket = &buckets[4 * k];
+
+ fast_sint_t c;
+ (void)(threads);
+
+ for (c = (fast_sint_t)k - 1; c >= 1; c -= 1) {
+ fast_sint_t i, j;
+ sa_sint_t s = SAINT_MIN;
+ for (i = (fast_sint_t)buckets[BUCKETS_INDEX4(c, 0)] - 1,
+ j = (fast_sint_t)temp_bucket[BUCKETS_INDEX2(c - 1, 0)] + 3;
+ i >= j; i -= 4) {
+ prefetchw(&SA[i - prefetch_distance]);
+
+ sa_sint_t p0 = SA[i - 0], q0 = (p0 & SAINT_MIN) ^ s;
+ s = s ^ q0;
+ SA[i - 0] = p0 ^ q0;
+ sa_sint_t p1 = SA[i - 1], q1 = (p1 & SAINT_MIN) ^ s;
+ s = s ^ q1;
+ SA[i - 1] = p1 ^ q1;
+ sa_sint_t p2 = SA[i - 2], q2 = (p2 & SAINT_MIN) ^ s;
+ s = s ^ q2;
+ SA[i - 2] = p2 ^ q2;
+ sa_sint_t p3 = SA[i - 3], q3 = (p3 & SAINT_MIN) ^ s;
+ s = s ^ q3;
+ SA[i - 3] = p3 ^ q3;
+ }
+
+ for (j -= 3; i >= j; i -= 1) {
+ sa_sint_t p = SA[i], q = (p & SAINT_MIN) ^ s;
+ s = s ^ q;
+ SA[i] = p ^ q;
+ }
+ }
+}
+
+static void libsais_partial_sorting_shift_markers_32s_4k(sa_sint_t * RESTRICT SA, sa_sint_t n) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i;
+ sa_sint_t s = SUFFIX_GROUP_MARKER;
+ for (i = (fast_sint_t)n - 1; i >= 3; i -= 4) {
+ prefetchw(&SA[i - prefetch_distance]);
+
+ sa_sint_t p0 = SA[i - 0],
+ q0 = ((p0 & SUFFIX_GROUP_MARKER) ^ s) & ((sa_sint_t)(p0 > 0) << ((SUFFIX_GROUP_BIT - 1)));
+ s = s ^ q0;
+ SA[i - 0] = p0 ^ q0;
+ sa_sint_t p1 = SA[i - 1],
+ q1 = ((p1 & SUFFIX_GROUP_MARKER) ^ s) & ((sa_sint_t)(p1 > 0) << ((SUFFIX_GROUP_BIT - 1)));
+ s = s ^ q1;
+ SA[i - 1] = p1 ^ q1;
+ sa_sint_t p2 = SA[i - 2],
+ q2 = ((p2 & SUFFIX_GROUP_MARKER) ^ s) & ((sa_sint_t)(p2 > 0) << ((SUFFIX_GROUP_BIT - 1)));
+ s = s ^ q2;
+ SA[i - 2] = p2 ^ q2;
+ sa_sint_t p3 = SA[i - 3],
+ q3 = ((p3 & SUFFIX_GROUP_MARKER) ^ s) & ((sa_sint_t)(p3 > 0) << ((SUFFIX_GROUP_BIT - 1)));
+ s = s ^ q3;
+ SA[i - 3] = p3 ^ q3;
+ }
+
+ for (; i >= 0; i -= 1) {
+ sa_sint_t p = SA[i], q = ((p & SUFFIX_GROUP_MARKER) ^ s) & ((sa_sint_t)(p > 0) << ((SUFFIX_GROUP_BIT - 1)));
+ s = s ^ q;
+ SA[i] = p ^ q;
+ }
+}
+
+static void libsais_partial_sorting_shift_buckets_32s_6k(sa_sint_t k, sa_sint_t * RESTRICT buckets) {
+ sa_sint_t * RESTRICT temp_bucket = &buckets[4 * k];
+
+ fast_sint_t i;
+ for (i = BUCKETS_INDEX2(0, 0); i <= BUCKETS_INDEX2((fast_sint_t)k - 1, 0); i += BUCKETS_INDEX2(1, 0)) {
+ buckets[2 * i + BUCKETS_INDEX4(0, 0)] = temp_bucket[i + BUCKETS_INDEX2(0, 0)];
+ buckets[2 * i + BUCKETS_INDEX4(0, 1)] = temp_bucket[i + BUCKETS_INDEX2(0, 1)];
+ }
+}
+
+static sa_sint_t libsais_partial_sorting_scan_right_to_left_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT buckets, sa_sint_t d,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT induction_bucket = &buckets[0 * ALPHABET_SIZE];
+ sa_sint_t * RESTRICT distinct_names = &buckets[2 * ALPHABET_SIZE];
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + prefetch_distance + 1; i >= j; i -= 2) {
+ prefetch(&SA[i - 2 * prefetch_distance]);
+
+ prefetch(&T[SA[i - prefetch_distance - 0] & SAINT_MAX] - 1);
+ prefetch(&T[SA[i - prefetch_distance - 0] & SAINT_MAX] - 2);
+ prefetch(&T[SA[i - prefetch_distance - 1] & SAINT_MAX] - 1);
+ prefetch(&T[SA[i - prefetch_distance - 1] & SAINT_MAX] - 2);
+
+ sa_sint_t p0 = SA[i - 0];
+ d += (p0 < 0);
+ p0 &= SAINT_MAX;
+ sa_sint_t v0 = BUCKETS_INDEX2(T[p0 - 1], T[p0 - 2] > T[p0 - 1]);
+ SA[--induction_bucket[v0]] = (p0 - 1) | ((sa_sint_t)(distinct_names[v0] != d) << (SAINT_BIT - 1));
+ distinct_names[v0] = d;
+
+ sa_sint_t p1 = SA[i - 1];
+ d += (p1 < 0);
+ p1 &= SAINT_MAX;
+ sa_sint_t v1 = BUCKETS_INDEX2(T[p1 - 1], T[p1 - 2] > T[p1 - 1]);
+ SA[--induction_bucket[v1]] = (p1 - 1) | ((sa_sint_t)(distinct_names[v1] != d) << (SAINT_BIT - 1));
+ distinct_names[v1] = d;
+ }
+
+ for (j -= prefetch_distance + 1; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ d += (p < 0);
+ p &= SAINT_MAX;
+ sa_sint_t v = BUCKETS_INDEX2(T[p - 1], T[p - 2] > T[p - 1]);
+ SA[--induction_bucket[v]] = (p - 1) | ((sa_sint_t)(distinct_names[v] != d) << (SAINT_BIT - 1));
+ distinct_names[v] = d;
+ }
+
+ return d;
+}
+static void libsais_partial_sorting_scan_right_to_left_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix, sa_sint_t left_suffixes_count,
+ sa_sint_t d, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ fast_sint_t scan_start = (fast_sint_t)left_suffixes_count + 1;
+ fast_sint_t scan_end = (fast_sint_t)n - (fast_sint_t)first_lms_suffix;
+
+ if (threads == 1 || (scan_end - scan_start) < 65536) {
+ libsais_partial_sorting_scan_right_to_left_8u(T, SA, buckets, d, scan_start, scan_end - scan_start);
+ }
+ (void)(thread_state);
+}
+
+static sa_sint_t libsais_partial_sorting_scan_right_to_left_32s_6k(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT buckets, sa_sint_t d,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + 2 * prefetch_distance + 1; i >= j; i -= 2) {
+ prefetch(&SA[i - 3 * prefetch_distance]);
+
+ prefetch(&T[SA[i - 2 * prefetch_distance - 0] & SAINT_MAX] - 1);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 0] & SAINT_MAX] - 2);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 1] & SAINT_MAX] - 1);
+ prefetch(&T[SA[i - 2 * prefetch_distance - 1] & SAINT_MAX] - 2);
+
+ sa_sint_t p0 = SA[i - prefetch_distance - 0] & SAINT_MAX;
+ sa_sint_t v0 = BUCKETS_INDEX4(T[p0 - (p0 > 0)], 0);
+ prefetchw(&buckets[v0]);
+ sa_sint_t p1 = SA[i - prefetch_distance - 1] & SAINT_MAX;
+ sa_sint_t v1 = BUCKETS_INDEX4(T[p1 - (p1 > 0)], 0);
+ prefetchw(&buckets[v1]);
+
+ sa_sint_t p2 = SA[i - 0];
+ d += (p2 < 0);
+ p2 &= SAINT_MAX;
+ sa_sint_t v2 = BUCKETS_INDEX4(T[p2 - 1], T[p2 - 2] > T[p2 - 1]);
+ SA[--buckets[v2]] = (p2 - 1) | ((sa_sint_t)(buckets[2 + v2] != d) << (SAINT_BIT - 1));
+ buckets[2 + v2] = d;
+
+ sa_sint_t p3 = SA[i - 1];
+ d += (p3 < 0);
+ p3 &= SAINT_MAX;
+ sa_sint_t v3 = BUCKETS_INDEX4(T[p3 - 1], T[p3 - 2] > T[p3 - 1]);
+ SA[--buckets[v3]] = (p3 - 1) | ((sa_sint_t)(buckets[2 + v3] != d) << (SAINT_BIT - 1));
+ buckets[2 + v3] = d;
+ }
+
+ for (j -= 2 * prefetch_distance + 1; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ d += (p < 0);
+ p &= SAINT_MAX;
+ sa_sint_t v = BUCKETS_INDEX4(T[p - 1], T[p - 2] > T[p - 1]);
+ SA[--buckets[v]] = (p - 1) | ((sa_sint_t)(buckets[2 + v] != d) << (SAINT_BIT - 1));
+ buckets[2 + v] = d;
+ }
+
+ return d;
+}
+
+static sa_sint_t libsais_partial_sorting_scan_right_to_left_32s_4k(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t k,
+ sa_sint_t * RESTRICT buckets, sa_sint_t d,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT induction_bucket = &buckets[3 * k];
+ sa_sint_t * RESTRICT distinct_names = &buckets[0 * k];
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + 2 * prefetch_distance + 1; i >= j; i -= 2) {
+ prefetchw(&SA[i - 3 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i - 2 * prefetch_distance - 0];
+ const sa_sint_t * Ts0 = &T[s0 & ~SUFFIX_GROUP_MARKER] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ Ts0--;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i - 2 * prefetch_distance - 1];
+ const sa_sint_t * Ts1 = &T[s1 & ~SUFFIX_GROUP_MARKER] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ Ts1--;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ sa_sint_t s2 = SA[i - 1 * prefetch_distance - 0];
+ if (s2 > 0) {
+ const fast_sint_t Ts2 = T[(s2 & ~SUFFIX_GROUP_MARKER) - 1];
+ prefetchw(&induction_bucket[Ts2]);
+ prefetchw(&distinct_names[BUCKETS_INDEX2(Ts2, 0)]);
+ }
+ sa_sint_t s3 = SA[i - 1 * prefetch_distance - 1];
+ if (s3 > 0) {
+ const fast_sint_t Ts3 = T[(s3 & ~SUFFIX_GROUP_MARKER) - 1];
+ prefetchw(&induction_bucket[Ts3]);
+ prefetchw(&distinct_names[BUCKETS_INDEX2(Ts3, 0)]);
+ }
+
+ sa_sint_t p0 = SA[i - 0];
+ if (p0 > 0) {
+ SA[i - 0] = 0;
+ d += (p0 >> (SUFFIX_GROUP_BIT - 1));
+ p0 &= ~SUFFIX_GROUP_MARKER;
+ sa_sint_t v0 = BUCKETS_INDEX2(T[p0 - 1], T[p0 - 2] > T[p0 - 1]);
+ SA[--induction_bucket[T[p0 - 1]]] = (p0 - 1) | ((sa_sint_t)(T[p0 - 2] > T[p0 - 1]) << (SAINT_BIT - 1)) |
+ ((sa_sint_t)(distinct_names[v0] != d) << (SUFFIX_GROUP_BIT - 1));
+ distinct_names[v0] = d;
+ }
+
+ sa_sint_t p1 = SA[i - 1];
+ if (p1 > 0) {
+ SA[i - 1] = 0;
+ d += (p1 >> (SUFFIX_GROUP_BIT - 1));
+ p1 &= ~SUFFIX_GROUP_MARKER;
+ sa_sint_t v1 = BUCKETS_INDEX2(T[p1 - 1], T[p1 - 2] > T[p1 - 1]);
+ SA[--induction_bucket[T[p1 - 1]]] = (p1 - 1) | ((sa_sint_t)(T[p1 - 2] > T[p1 - 1]) << (SAINT_BIT - 1)) |
+ ((sa_sint_t)(distinct_names[v1] != d) << (SUFFIX_GROUP_BIT - 1));
+ distinct_names[v1] = d;
+ }
+ }
+
+ for (j -= 2 * prefetch_distance + 1; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ if (p > 0) {
+ SA[i] = 0;
+ d += (p >> (SUFFIX_GROUP_BIT - 1));
+ p &= ~SUFFIX_GROUP_MARKER;
+ sa_sint_t v = BUCKETS_INDEX2(T[p - 1], T[p - 2] > T[p - 1]);
+ SA[--induction_bucket[T[p - 1]]] = (p - 1) | ((sa_sint_t)(T[p - 2] > T[p - 1]) << (SAINT_BIT - 1)) |
+ ((sa_sint_t)(distinct_names[v] != d) << (SUFFIX_GROUP_BIT - 1));
+ distinct_names[v] = d;
+ }
+ }
+
+ return d;
+}
+
+static void libsais_partial_sorting_scan_right_to_left_32s_1k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + 2 * prefetch_distance + 1; i >= j; i -= 2) {
+ prefetchw(&SA[i - 3 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i - 2 * prefetch_distance - 0];
+ const sa_sint_t * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i - 2 * prefetch_distance - 1];
+ const sa_sint_t * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ sa_sint_t s2 = SA[i - 1 * prefetch_distance - 0];
+ if (s2 > 0) {
+ prefetchw(&induction_bucket[T[s2 - 1]]);
+ prefetch(&T[s2] - 2);
+ }
+ sa_sint_t s3 = SA[i - 1 * prefetch_distance - 1];
+ if (s3 > 0) {
+ prefetchw(&induction_bucket[T[s3 - 1]]);
+ prefetch(&T[s3] - 2);
+ }
+
+ sa_sint_t p0 = SA[i - 0];
+ if (p0 > 0) {
+ SA[i - 0] = 0;
+ SA[--induction_bucket[T[p0 - 1]]] = (p0 - 1) | ((sa_sint_t)(T[p0 - 2] > T[p0 - 1]) << (SAINT_BIT - 1));
+ }
+ sa_sint_t p1 = SA[i - 1];
+ if (p1 > 0) {
+ SA[i - 1] = 0;
+ SA[--induction_bucket[T[p1 - 1]]] = (p1 - 1) | ((sa_sint_t)(T[p1 - 2] > T[p1 - 1]) << (SAINT_BIT - 1));
+ }
+ }
+
+ for (j -= 2 * prefetch_distance + 1; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ if (p > 0) {
+ SA[i] = 0;
+ SA[--induction_bucket[T[p - 1]]] = (p - 1) | ((sa_sint_t)(T[p - 2] > T[p - 1]) << (SAINT_BIT - 1));
+ }
+ }
+}
+static sa_sint_t libsais_partial_sorting_scan_right_to_left_32s_6k_omp(
+ const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix, sa_sint_t left_suffixes_count, sa_sint_t d, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ fast_sint_t scan_start = (fast_sint_t)left_suffixes_count + 1;
+ fast_sint_t scan_end = (fast_sint_t)n - (fast_sint_t)first_lms_suffix;
+
+ if (threads == 1 || (scan_end - scan_start) < 65536) {
+ d = libsais_partial_sorting_scan_right_to_left_32s_6k(T, SA, buckets, d, scan_start, scan_end - scan_start);
+ }
+ (void)(thread_state);
+ return d;
+}
+
+static sa_sint_t libsais_partial_sorting_scan_right_to_left_32s_4k_omp(const sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ sa_sint_t d, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (threads == 1 || n < 65536) {
+ d = libsais_partial_sorting_scan_right_to_left_32s_4k(T, SA, k, buckets, d, 0, n);
+ }
+ (void)(thread_state);
+ return d;
+}
+
+static void libsais_partial_sorting_scan_right_to_left_32s_1k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT buckets,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (threads == 1 || n < 65536) {
+ libsais_partial_sorting_scan_right_to_left_32s_1k(T, SA, buckets, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static fast_sint_t libsais_partial_sorting_gather_lms_suffixes_32s_4k(sa_sint_t * RESTRICT SA,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j, l;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - 3, l = omp_block_start; i < j; i += 4) {
+ prefetch(&SA[i + prefetch_distance]);
+
+ sa_sint_t s0 = SA[i + 0];
+ SA[l] = (s0 - SUFFIX_GROUP_MARKER) & (~SUFFIX_GROUP_MARKER);
+ l += (s0 < 0);
+ sa_sint_t s1 = SA[i + 1];
+ SA[l] = (s1 - SUFFIX_GROUP_MARKER) & (~SUFFIX_GROUP_MARKER);
+ l += (s1 < 0);
+ sa_sint_t s2 = SA[i + 2];
+ SA[l] = (s2 - SUFFIX_GROUP_MARKER) & (~SUFFIX_GROUP_MARKER);
+ l += (s2 < 0);
+ sa_sint_t s3 = SA[i + 3];
+ SA[l] = (s3 - SUFFIX_GROUP_MARKER) & (~SUFFIX_GROUP_MARKER);
+ l += (s3 < 0);
+ }
+
+ for (j += 3; i < j; i += 1) {
+ sa_sint_t s = SA[i];
+ SA[l] = (s - SUFFIX_GROUP_MARKER) & (~SUFFIX_GROUP_MARKER);
+ l += (s < 0);
+ }
+
+ return l;
+}
+
+static fast_sint_t libsais_partial_sorting_gather_lms_suffixes_32s_1k(sa_sint_t * RESTRICT SA,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j, l;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - 3, l = omp_block_start; i < j; i += 4) {
+ prefetch(&SA[i + prefetch_distance]);
+
+ sa_sint_t s0 = SA[i + 0];
+ SA[l] = s0 & SAINT_MAX;
+ l += (s0 < 0);
+ sa_sint_t s1 = SA[i + 1];
+ SA[l] = s1 & SAINT_MAX;
+ l += (s1 < 0);
+ sa_sint_t s2 = SA[i + 2];
+ SA[l] = s2 & SAINT_MAX;
+ l += (s2 < 0);
+ sa_sint_t s3 = SA[i + 3];
+ SA[l] = s3 & SAINT_MAX;
+ l += (s3 < 0);
+ }
+
+ for (j += 3; i < j; i += 1) {
+ sa_sint_t s = SA[i];
+ SA[l] = s & SAINT_MAX;
+ l += (s < 0);
+ }
+
+ return l;
+}
+
+static void libsais_partial_sorting_gather_lms_suffixes_32s_4k_omp(sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (n / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : n - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ libsais_partial_sorting_gather_lms_suffixes_32s_4k(SA, omp_block_start, omp_block_size);
+ }
+ }
+}
+
+static void libsais_partial_sorting_gather_lms_suffixes_32s_1k_omp(sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (n / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : n - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ libsais_partial_sorting_gather_lms_suffixes_32s_1k(SA, omp_block_start, omp_block_size);
+ }
+ }
+}
+
+static void libsais_induce_partial_order_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t * RESTRICT buckets, sa_sint_t first_lms_suffix,
+ sa_sint_t left_suffixes_count, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ memset(&buckets[2 * ALPHABET_SIZE], 0, 2 * ALPHABET_SIZE * sizeof(sa_sint_t));
+
+ sa_sint_t d = libsais_partial_sorting_scan_left_to_right_8u_omp(T, SA, n, buckets, left_suffixes_count, 0, threads,
+ thread_state);
+ libsais_partial_sorting_shift_markers_8u_omp(SA, n, buckets, threads);
+ libsais_partial_sorting_scan_right_to_left_8u_omp(T, SA, n, buckets, first_lms_suffix, left_suffixes_count, d,
+ threads, thread_state);
+}
+
+static void libsais_induce_partial_order_32s_6k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets,
+ sa_sint_t first_lms_suffix, sa_sint_t left_suffixes_count,
+ sa_sint_t threads, LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t d = libsais_partial_sorting_scan_left_to_right_32s_6k_omp(T, SA, n, buckets, left_suffixes_count, 0,
+ threads, thread_state);
+ libsais_partial_sorting_shift_markers_32s_6k_omp(SA, k, buckets, threads);
+ libsais_partial_sorting_shift_buckets_32s_6k(k, buckets);
+ libsais_partial_sorting_scan_right_to_left_32s_6k_omp(T, SA, n, buckets, first_lms_suffix, left_suffixes_count, d,
+ threads, thread_state);
+}
+
+static void libsais_induce_partial_order_32s_4k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ memset(buckets, 0, 2 * (size_t)k * sizeof(sa_sint_t));
+
+ sa_sint_t d = libsais_partial_sorting_scan_left_to_right_32s_4k_omp(T, SA, n, k, buckets, 0, threads, thread_state);
+ libsais_partial_sorting_shift_markers_32s_4k(SA, n);
+ libsais_partial_sorting_scan_right_to_left_32s_4k_omp(T, SA, n, k, buckets, d, threads, thread_state);
+ libsais_partial_sorting_gather_lms_suffixes_32s_4k_omp(SA, n, threads, thread_state);
+}
+
+static void libsais_induce_partial_order_32s_2k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ libsais_partial_sorting_scan_left_to_right_32s_1k_omp(T, SA, n, &buckets[1 * k], threads, thread_state);
+ libsais_partial_sorting_scan_right_to_left_32s_1k_omp(T, SA, n, &buckets[0 * k], threads, thread_state);
+ libsais_partial_sorting_gather_lms_suffixes_32s_1k_omp(SA, n, threads, thread_state);
+}
+
+static void libsais_induce_partial_order_32s_1k_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ libsais_count_suffixes_32s(T, n, k, buckets);
+ libsais_initialize_buckets_start_32s_1k(k, buckets);
+ libsais_partial_sorting_scan_left_to_right_32s_1k_omp(T, SA, n, buckets, threads, thread_state);
+
+ libsais_count_suffixes_32s(T, n, k, buckets);
+ libsais_initialize_buckets_end_32s_1k(k, buckets);
+ libsais_partial_sorting_scan_right_to_left_32s_1k_omp(T, SA, n, buckets, threads, thread_state);
+
+ libsais_partial_sorting_gather_lms_suffixes_32s_1k_omp(SA, n, threads, thread_state);
+}
+
+static sa_sint_t libsais_renumber_lms_suffixes_8u(sa_sint_t * RESTRICT SA, sa_sint_t m, sa_sint_t name,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT SAm = &SA[m];
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 3; i < j; i += 4) {
+ prefetch(&SA[i + 2 * prefetch_distance]);
+
+ prefetchw(&SAm[(SA[i + prefetch_distance + 0] & SAINT_MAX) >> 1]);
+ prefetchw(&SAm[(SA[i + prefetch_distance + 1] & SAINT_MAX) >> 1]);
+ prefetchw(&SAm[(SA[i + prefetch_distance + 2] & SAINT_MAX) >> 1]);
+ prefetchw(&SAm[(SA[i + prefetch_distance + 3] & SAINT_MAX) >> 1]);
+
+ sa_sint_t p0 = SA[i + 0];
+ SAm[(p0 & SAINT_MAX) >> 1] = name | SAINT_MIN;
+ name += p0 < 0;
+ sa_sint_t p1 = SA[i + 1];
+ SAm[(p1 & SAINT_MAX) >> 1] = name | SAINT_MIN;
+ name += p1 < 0;
+ sa_sint_t p2 = SA[i + 2];
+ SAm[(p2 & SAINT_MAX) >> 1] = name | SAINT_MIN;
+ name += p2 < 0;
+ sa_sint_t p3 = SA[i + 3];
+ SAm[(p3 & SAINT_MAX) >> 1] = name | SAINT_MIN;
+ name += p3 < 0;
+ }
+
+ for (j += prefetch_distance + 3; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ SAm[(p & SAINT_MAX) >> 1] = name | SAINT_MIN;
+ name += p < 0;
+ }
+
+ return name;
+}
+
+static fast_sint_t libsais_gather_marked_suffixes_8u(sa_sint_t * RESTRICT SA, sa_sint_t m, fast_sint_t l,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ l -= 1;
+
+ fast_sint_t i, j;
+ for (i = (fast_sint_t)m + omp_block_start + omp_block_size - 1, j = (fast_sint_t)m + omp_block_start + 3; i >= j;
+ i -= 4) {
+ prefetch(&SA[i - prefetch_distance]);
+
+ sa_sint_t s0 = SA[i - 0];
+ SA[l] = s0 & SAINT_MAX;
+ l -= s0 < 0;
+ sa_sint_t s1 = SA[i - 1];
+ SA[l] = s1 & SAINT_MAX;
+ l -= s1 < 0;
+ sa_sint_t s2 = SA[i - 2];
+ SA[l] = s2 & SAINT_MAX;
+ l -= s2 < 0;
+ sa_sint_t s3 = SA[i - 3];
+ SA[l] = s3 & SAINT_MAX;
+ l -= s3 < 0;
+ }
+
+ for (j -= 3; i >= j; i -= 1) {
+ sa_sint_t s = SA[i];
+ SA[l] = s & SAINT_MAX;
+ l -= s < 0;
+ }
+
+ l += 1;
+
+ return l;
+}
+
+static sa_sint_t libsais_renumber_lms_suffixes_8u_omp(sa_sint_t * RESTRICT SA, sa_sint_t m, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t name = 0;
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (m / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : m - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ name = libsais_renumber_lms_suffixes_8u(SA, m, 0, omp_block_start, omp_block_size);
+ }
+ }
+
+ return name;
+}
+
+static void libsais_gather_marked_lms_suffixes_8u_omp(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m, sa_sint_t fs,
+ sa_sint_t threads, LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (((fast_sint_t)n >> 1) / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size =
+ omp_thread_num < omp_num_threads - 1 ? omp_block_stride : ((fast_sint_t)n >> 1) - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ libsais_gather_marked_suffixes_8u(SA, m, (fast_sint_t)n + (fast_sint_t)fs, omp_block_start, omp_block_size);
+ }
+ }
+}
+
+static sa_sint_t libsais_renumber_and_gather_lms_suffixes_8u_omp(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m,
+ sa_sint_t fs, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ memset(&SA[m], 0, ((size_t)n >> 1) * sizeof(sa_sint_t));
+
+ sa_sint_t name = libsais_renumber_lms_suffixes_8u_omp(SA, m, threads, thread_state);
+ if (name < m) {
+ libsais_gather_marked_lms_suffixes_8u_omp(SA, n, m, fs, threads, thread_state);
+ } else {
+ fast_sint_t i;
+ for (i = 0; i < m; i += 1) {
+ SA[i] &= SAINT_MAX;
+ }
+ }
+
+ return name;
+}
+
+static sa_sint_t libsais_renumber_distinct_lms_suffixes_32s_4k(sa_sint_t * RESTRICT SA, sa_sint_t m, sa_sint_t name,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT SAm = &SA[m];
+
+ fast_sint_t i, j;
+ sa_sint_t p0, p1, p2, p3 = 0;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 3; i < j; i += 4) {
+ prefetchw(&SA[i + 2 * prefetch_distance]);
+
+ prefetchw(&SAm[(SA[i + prefetch_distance + 0] & SAINT_MAX) >> 1]);
+ prefetchw(&SAm[(SA[i + prefetch_distance + 1] & SAINT_MAX) >> 1]);
+ prefetchw(&SAm[(SA[i + prefetch_distance + 2] & SAINT_MAX) >> 1]);
+ prefetchw(&SAm[(SA[i + prefetch_distance + 3] & SAINT_MAX) >> 1]);
+
+ p0 = SA[i + 0];
+ SAm[(SA[i + 0] = p0 & SAINT_MAX) >> 1] = name | (p0 & p3 & SAINT_MIN);
+ name += p0 < 0;
+ p1 = SA[i + 1];
+ SAm[(SA[i + 1] = p1 & SAINT_MAX) >> 1] = name | (p1 & p0 & SAINT_MIN);
+ name += p1 < 0;
+ p2 = SA[i + 2];
+ SAm[(SA[i + 2] = p2 & SAINT_MAX) >> 1] = name | (p2 & p1 & SAINT_MIN);
+ name += p2 < 0;
+ p3 = SA[i + 3];
+ SAm[(SA[i + 3] = p3 & SAINT_MAX) >> 1] = name | (p3 & p2 & SAINT_MIN);
+ name += p3 < 0;
+ }
+
+ for (j += prefetch_distance + 3; i < j; i += 1) {
+ p2 = p3;
+ p3 = SA[i];
+ SAm[(SA[i] = p3 & SAINT_MAX) >> 1] = name | (p3 & p2 & SAINT_MIN);
+ name += p3 < 0;
+ }
+
+ return name;
+}
+
+static void libsais_mark_distinct_lms_suffixes_32s(sa_sint_t * RESTRICT SA, sa_sint_t m, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ sa_sint_t p0, p1, p2, p3 = 0;
+ for (i = (fast_sint_t)m + omp_block_start, j = (fast_sint_t)m + omp_block_start + omp_block_size - 3; i < j;
+ i += 4) {
+ prefetchw(&SA[i + prefetch_distance]);
+
+ p0 = SA[i + 0];
+ SA[i + 0] = p0 & (p3 | SAINT_MAX);
+ p0 = (p0 == 0) ? p3 : p0;
+ p1 = SA[i + 1];
+ SA[i + 1] = p1 & (p0 | SAINT_MAX);
+ p1 = (p1 == 0) ? p0 : p1;
+ p2 = SA[i + 2];
+ SA[i + 2] = p2 & (p1 | SAINT_MAX);
+ p2 = (p2 == 0) ? p1 : p2;
+ p3 = SA[i + 3];
+ SA[i + 3] = p3 & (p2 | SAINT_MAX);
+ p3 = (p3 == 0) ? p2 : p3;
+ }
+
+ for (j += 3; i < j; i += 1) {
+ p2 = p3;
+ p3 = SA[i];
+ SA[i] = p3 & (p2 | SAINT_MAX);
+ p3 = (p3 == 0) ? p2 : p3;
+ }
+}
+
+static void libsais_clamp_lms_suffixes_length_32s(sa_sint_t * RESTRICT SA, sa_sint_t m, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT SAm = &SA[m];
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - 3; i < j; i += 4) {
+ prefetchw(&SAm[i + prefetch_distance]);
+
+ SAm[i + 0] = (SAm[i + 0] < 0 ? SAm[i + 0] : 0) & SAINT_MAX;
+ SAm[i + 1] = (SAm[i + 1] < 0 ? SAm[i + 1] : 0) & SAINT_MAX;
+ SAm[i + 2] = (SAm[i + 2] < 0 ? SAm[i + 2] : 0) & SAINT_MAX;
+ SAm[i + 3] = (SAm[i + 3] < 0 ? SAm[i + 3] : 0) & SAINT_MAX;
+ }
+
+ for (j += 3; i < j; i += 1) {
+ SAm[i] = (SAm[i] < 0 ? SAm[i] : 0) & SAINT_MAX;
+ }
+}
+
+static sa_sint_t libsais_renumber_distinct_lms_suffixes_32s_4k_omp(sa_sint_t * RESTRICT SA, sa_sint_t m,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t name = 0;
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (m / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : m - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ name = libsais_renumber_distinct_lms_suffixes_32s_4k(SA, m, 1, omp_block_start, omp_block_size);
+ }
+ }
+
+ return name - 1;
+}
+
+static void libsais_mark_distinct_lms_suffixes_32s_omp(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m,
+ sa_sint_t threads) {
+ {
+ (void)(threads);
+
+ fast_sint_t omp_block_start = 0;
+ fast_sint_t omp_block_size = (fast_sint_t)n >> 1;
+
+ libsais_mark_distinct_lms_suffixes_32s(SA, m, omp_block_start, omp_block_size);
+ }
+}
+
+static void libsais_clamp_lms_suffixes_length_32s_omp(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m,
+ sa_sint_t threads) {
+ {
+ (void)(threads);
+
+ fast_sint_t omp_block_start = 0;
+ fast_sint_t omp_block_size = (fast_sint_t)n >> 1;
+
+ libsais_clamp_lms_suffixes_length_32s(SA, m, omp_block_start, omp_block_size);
+ }
+}
+
+static sa_sint_t libsais_renumber_and_mark_distinct_lms_suffixes_32s_4k_omp(
+ sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ memset(&SA[m], 0, ((size_t)n >> 1) * sizeof(sa_sint_t));
+
+ sa_sint_t name = libsais_renumber_distinct_lms_suffixes_32s_4k_omp(SA, m, threads, thread_state);
+ if (name < m) {
+ libsais_mark_distinct_lms_suffixes_32s_omp(SA, n, m, threads);
+ }
+
+ return name;
+}
+
+static sa_sint_t libsais_renumber_and_mark_distinct_lms_suffixes_32s_1k_omp(sa_sint_t * RESTRICT T,
+ sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t m, sa_sint_t threads) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT SAm = &SA[m];
+
+ {
+ libsais_gather_lms_suffixes_32s(T, SA, n);
+
+ memset(&SA[m], 0, ((size_t)n - (size_t)m - (size_t)m) * sizeof(sa_sint_t));
+
+ fast_sint_t i, j;
+ for (i = (fast_sint_t)n - (fast_sint_t)m, j = (fast_sint_t)n - 1 - prefetch_distance - 3; i < j; i += 4) {
+ prefetch(&SA[i + 2 * prefetch_distance]);
+
+ prefetchw(&SAm[((sa_uint_t)SA[i + prefetch_distance + 0]) >> 1]);
+ prefetchw(&SAm[((sa_uint_t)SA[i + prefetch_distance + 1]) >> 1]);
+ prefetchw(&SAm[((sa_uint_t)SA[i + prefetch_distance + 2]) >> 1]);
+ prefetchw(&SAm[((sa_uint_t)SA[i + prefetch_distance + 3]) >> 1]);
+
+ SAm[((sa_uint_t)SA[i + 0]) >> 1] = SA[i + 1] - SA[i + 0] + 1 + SAINT_MIN;
+ SAm[((sa_uint_t)SA[i + 1]) >> 1] = SA[i + 2] - SA[i + 1] + 1 + SAINT_MIN;
+ SAm[((sa_uint_t)SA[i + 2]) >> 1] = SA[i + 3] - SA[i + 2] + 1 + SAINT_MIN;
+ SAm[((sa_uint_t)SA[i + 3]) >> 1] = SA[i + 4] - SA[i + 3] + 1 + SAINT_MIN;
+ }
+
+ for (j += prefetch_distance + 3; i < j; i += 1) {
+ SAm[((sa_uint_t)SA[i]) >> 1] = SA[i + 1] - SA[i] + 1 + SAINT_MIN;
+ }
+
+ SAm[((sa_uint_t)SA[n - 1]) >> 1] = 1 + SAINT_MIN;
+ }
+
+ { libsais_clamp_lms_suffixes_length_32s_omp(SA, n, m, threads); }
+
+ sa_sint_t name = 1;
+
+ {
+ fast_sint_t i, j, p = SA[0], plen = SAm[p >> 1];
+ sa_sint_t pdiff = SAINT_MIN;
+ for (i = 1, j = m - prefetch_distance - 1; i < j; i += 2) {
+ prefetch(&SA[i + 2 * prefetch_distance]);
+
+ prefetchw(&SAm[((sa_uint_t)SA[i + prefetch_distance + 0]) >> 1]);
+ prefetch(&T[((sa_uint_t)SA[i + prefetch_distance + 0])]);
+ prefetchw(&SAm[((sa_uint_t)SA[i + prefetch_distance + 1]) >> 1]);
+ prefetch(&T[((sa_uint_t)SA[i + prefetch_distance + 1])]);
+
+ fast_sint_t q = SA[i + 0], qlen = SAm[q >> 1];
+ sa_sint_t qdiff = SAINT_MIN;
+ if (plen == qlen) {
+ fast_sint_t l = 0;
+ do {
+ if (T[p + l] != T[q + l]) {
+ break;
+ }
+ } while (++l < qlen);
+ qdiff = (sa_sint_t)(l - qlen) & SAINT_MIN;
+ }
+ SAm[p >> 1] = name | (pdiff & qdiff);
+ name += (qdiff < 0);
+
+ p = SA[i + 1];
+ plen = SAm[p >> 1];
+ pdiff = SAINT_MIN;
+ if (qlen == plen) {
+ fast_sint_t l = 0;
+ do {
+ if (T[q + l] != T[p + l]) {
+ break;
+ }
+ } while (++l < plen);
+ pdiff = (sa_sint_t)(l - plen) & SAINT_MIN;
+ }
+ SAm[q >> 1] = name | (qdiff & pdiff);
+ name += (pdiff < 0);
+ }
+
+ for (j += prefetch_distance + 1; i < j; i += 1) {
+ fast_sint_t q = SA[i], qlen = SAm[q >> 1];
+ sa_sint_t qdiff = SAINT_MIN;
+ if (plen == qlen) {
+ fast_sint_t l = 0;
+ do {
+ if (T[p + l] != T[q + l]) {
+ break;
+ }
+ } while (++l < plen);
+ qdiff = (sa_sint_t)(l - plen) & SAINT_MIN;
+ }
+ SAm[p >> 1] = name | (pdiff & qdiff);
+ name += (qdiff < 0);
+
+ p = q;
+ plen = qlen;
+ pdiff = qdiff;
+ }
+
+ SAm[p >> 1] = name | pdiff;
+ name++;
+ }
+
+ if (name <= m) {
+ libsais_mark_distinct_lms_suffixes_32s_omp(SA, n, m, threads);
+ }
+
+ return name - 1;
+}
+
+static void libsais_reconstruct_lms_suffixes(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ const sa_sint_t * RESTRICT SAnm = &SA[n - m];
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 3; i < j; i += 4) {
+ prefetchw(&SA[i + 2 * prefetch_distance]);
+
+ prefetch(&SAnm[SA[i + prefetch_distance + 0]]);
+ prefetch(&SAnm[SA[i + prefetch_distance + 1]]);
+ prefetch(&SAnm[SA[i + prefetch_distance + 2]]);
+ prefetch(&SAnm[SA[i + prefetch_distance + 3]]);
+
+ SA[i + 0] = SAnm[SA[i + 0]];
+ SA[i + 1] = SAnm[SA[i + 1]];
+ SA[i + 2] = SAnm[SA[i + 2]];
+ SA[i + 3] = SAnm[SA[i + 3]];
+ }
+
+ for (j += prefetch_distance + 3; i < j; i += 1) {
+ SA[i] = SAnm[SA[i]];
+ }
+}
+
+static void libsais_reconstruct_lms_suffixes_omp(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m, sa_sint_t threads) {
+ {
+ (void)(threads);
+
+ fast_sint_t omp_block_start = 0;
+ fast_sint_t omp_block_size = m;
+ libsais_reconstruct_lms_suffixes(SA, n, m, omp_block_start, omp_block_size);
+ }
+}
+
+static void libsais_place_lms_suffixes_interval_8u(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m,
+ const sa_sint_t * RESTRICT buckets) {
+ const sa_sint_t * RESTRICT bucket_end = &buckets[7 * ALPHABET_SIZE];
+
+ fast_sint_t c, j = n;
+ for (c = ALPHABET_SIZE - 2; c >= 0; --c) {
+ fast_sint_t l = (fast_sint_t)buckets[BUCKETS_INDEX2(c, 1) + BUCKETS_INDEX2(1, 0)] -
+ (fast_sint_t)buckets[BUCKETS_INDEX2(c, 1)];
+ if (l > 0) {
+ fast_sint_t i = bucket_end[c];
+ if (j - i > 0) {
+ memset(&SA[i], 0, (size_t)(j - i) * sizeof(sa_sint_t));
+ }
+
+ memmove(&SA[j = (i - l)], &SA[m -= (sa_sint_t)l], (size_t)l * sizeof(sa_sint_t));
+ }
+ }
+
+ memset(&SA[0], 0, (size_t)j * sizeof(sa_sint_t));
+}
+
+static void libsais_place_lms_suffixes_interval_32s_4k(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t k, sa_sint_t m,
+ const sa_sint_t * RESTRICT buckets) {
+ const sa_sint_t * RESTRICT bucket_end = &buckets[3 * k];
+
+ fast_sint_t c, j = n;
+ for (c = (fast_sint_t)k - 2; c >= 0; --c) {
+ fast_sint_t l = (fast_sint_t)buckets[BUCKETS_INDEX2(c, 1) + BUCKETS_INDEX2(1, 0)] -
+ (fast_sint_t)buckets[BUCKETS_INDEX2(c, 1)];
+ if (l > 0) {
+ fast_sint_t i = bucket_end[c];
+ if (j - i > 0) {
+ memset(&SA[i], 0, (size_t)(j - i) * sizeof(sa_sint_t));
+ }
+
+ memmove(&SA[j = (i - l)], &SA[m -= (sa_sint_t)l], (size_t)l * sizeof(sa_sint_t));
+ }
+ }
+
+ memset(&SA[0], 0, (size_t)j * sizeof(sa_sint_t));
+}
+
+static void libsais_place_lms_suffixes_interval_32s_2k(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t k, sa_sint_t m,
+ const sa_sint_t * RESTRICT buckets) {
+ fast_sint_t j = n;
+
+ if (k > 1) {
+ fast_sint_t c;
+ for (c = BUCKETS_INDEX2((fast_sint_t)k - 2, 0); c >= BUCKETS_INDEX2(0, 0); c -= BUCKETS_INDEX2(1, 0)) {
+ fast_sint_t l =
+ (fast_sint_t)buckets[c + BUCKETS_INDEX2(1, 1)] - (fast_sint_t)buckets[c + BUCKETS_INDEX2(0, 1)];
+ if (l > 0) {
+ fast_sint_t i = buckets[c];
+ if (j - i > 0) {
+ memset(&SA[i], 0, (size_t)(j - i) * sizeof(sa_sint_t));
+ }
+
+ memmove(&SA[j = (i - l)], &SA[m -= (sa_sint_t)l], (size_t)l * sizeof(sa_sint_t));
+ }
+ }
+ }
+
+ memset(&SA[0], 0, (size_t)j * sizeof(sa_sint_t));
+}
+
+static void libsais_place_lms_suffixes_interval_32s_1k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t k, sa_sint_t m, sa_sint_t * RESTRICT buckets) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t c = k - 1;
+ fast_sint_t i, l = buckets[c];
+ for (i = (fast_sint_t)m - 1; i >= prefetch_distance + 3; i -= 4) {
+ prefetch(&SA[i - 2 * prefetch_distance]);
+
+ prefetch(&T[SA[i - prefetch_distance - 0]]);
+ prefetch(&T[SA[i - prefetch_distance - 1]]);
+ prefetch(&T[SA[i - prefetch_distance - 2]]);
+ prefetch(&T[SA[i - prefetch_distance - 3]]);
+
+ sa_sint_t p0 = SA[i - 0];
+ if (T[p0] != c) {
+ c = T[p0];
+ memset(&SA[buckets[c]], 0, (size_t)(l - buckets[c]) * sizeof(sa_sint_t));
+ l = buckets[c];
+ }
+ SA[--l] = p0;
+ sa_sint_t p1 = SA[i - 1];
+ if (T[p1] != c) {
+ c = T[p1];
+ memset(&SA[buckets[c]], 0, (size_t)(l - buckets[c]) * sizeof(sa_sint_t));
+ l = buckets[c];
+ }
+ SA[--l] = p1;
+ sa_sint_t p2 = SA[i - 2];
+ if (T[p2] != c) {
+ c = T[p2];
+ memset(&SA[buckets[c]], 0, (size_t)(l - buckets[c]) * sizeof(sa_sint_t));
+ l = buckets[c];
+ }
+ SA[--l] = p2;
+ sa_sint_t p3 = SA[i - 3];
+ if (T[p3] != c) {
+ c = T[p3];
+ memset(&SA[buckets[c]], 0, (size_t)(l - buckets[c]) * sizeof(sa_sint_t));
+ l = buckets[c];
+ }
+ SA[--l] = p3;
+ }
+
+ for (; i >= 0; i -= 1) {
+ sa_sint_t p = SA[i];
+ if (T[p] != c) {
+ c = T[p];
+ memset(&SA[buckets[c]], 0, (size_t)(l - buckets[c]) * sizeof(sa_sint_t));
+ l = buckets[c];
+ }
+ SA[--l] = p;
+ }
+
+ memset(&SA[0], 0, (size_t)l * sizeof(sa_sint_t));
+}
+
+static void libsais_place_lms_suffixes_histogram_32s_6k(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t k, sa_sint_t m,
+ const sa_sint_t * RESTRICT buckets) {
+ const sa_sint_t * RESTRICT bucket_end = &buckets[5 * k];
+
+ fast_sint_t c, j = n;
+ for (c = (fast_sint_t)k - 2; c >= 0; --c) {
+ fast_sint_t l = (fast_sint_t)buckets[BUCKETS_INDEX4(c, 1)];
+ if (l > 0) {
+ fast_sint_t i = bucket_end[c];
+ if (j - i > 0) {
+ memset(&SA[i], 0, (size_t)(j - i) * sizeof(sa_sint_t));
+ }
+
+ memmove(&SA[j = (i - l)], &SA[m -= (sa_sint_t)l], (size_t)l * sizeof(sa_sint_t));
+ }
+ }
+
+ memset(&SA[0], 0, (size_t)j * sizeof(sa_sint_t));
+}
+
+static void libsais_place_lms_suffixes_histogram_32s_4k(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t k, sa_sint_t m,
+ const sa_sint_t * RESTRICT buckets) {
+ const sa_sint_t * RESTRICT bucket_end = &buckets[3 * k];
+
+ fast_sint_t c, j = n;
+ for (c = (fast_sint_t)k - 2; c >= 0; --c) {
+ fast_sint_t l = (fast_sint_t)buckets[BUCKETS_INDEX2(c, 1)];
+ if (l > 0) {
+ fast_sint_t i = bucket_end[c];
+ if (j - i > 0) {
+ memset(&SA[i], 0, (size_t)(j - i) * sizeof(sa_sint_t));
+ }
+
+ memmove(&SA[j = (i - l)], &SA[m -= (sa_sint_t)l], (size_t)l * sizeof(sa_sint_t));
+ }
+ }
+
+ memset(&SA[0], 0, (size_t)j * sizeof(sa_sint_t));
+}
+
+static void libsais_place_lms_suffixes_histogram_32s_2k(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t k, sa_sint_t m,
+ const sa_sint_t * RESTRICT buckets) {
+ fast_sint_t j = n;
+
+ if (k > 1) {
+ fast_sint_t c;
+ for (c = BUCKETS_INDEX2((fast_sint_t)k - 2, 0); c >= BUCKETS_INDEX2(0, 0); c -= BUCKETS_INDEX2(1, 0)) {
+ fast_sint_t l = (fast_sint_t)buckets[c + BUCKETS_INDEX2(0, 1)];
+ if (l > 0) {
+ fast_sint_t i = buckets[c];
+ if (j - i > 0) {
+ memset(&SA[i], 0, (size_t)(j - i) * sizeof(sa_sint_t));
+ }
+
+ memmove(&SA[j = (i - l)], &SA[m -= (sa_sint_t)l], (size_t)l * sizeof(sa_sint_t));
+ }
+ }
+ }
+
+ memset(&SA[0], 0, (size_t)j * sizeof(sa_sint_t));
+}
+
+static void libsais_final_bwt_scan_left_to_right_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 1; i < j; i += 2) {
+ prefetchw(&SA[i + 2 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i + prefetch_distance + 0];
+ const u8 * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ Ts0--;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i + prefetch_distance + 1];
+ const u8 * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ Ts1--;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+
+ sa_sint_t p0 = SA[i + 0];
+ SA[i + 0] = p0 & SAINT_MAX;
+ if (p0 > 0) {
+ p0--;
+ SA[i + 0] = T[p0] | SAINT_MIN;
+ SA[induction_bucket[T[p0]]++] = p0 | ((sa_sint_t)(T[p0 - (p0 > 0)] < T[p0]) << (SAINT_BIT - 1));
+ }
+ sa_sint_t p1 = SA[i + 1];
+ SA[i + 1] = p1 & SAINT_MAX;
+ if (p1 > 0) {
+ p1--;
+ SA[i + 1] = T[p1] | SAINT_MIN;
+ SA[induction_bucket[T[p1]]++] = p1 | ((sa_sint_t)(T[p1 - (p1 > 0)] < T[p1]) << (SAINT_BIT - 1));
+ }
+ }
+
+ for (j += prefetch_distance + 1; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p & SAINT_MAX;
+ if (p > 0) {
+ p--;
+ SA[i] = T[p] | SAINT_MIN;
+ SA[induction_bucket[T[p]]++] = p | ((sa_sint_t)(T[p - (p > 0)] < T[p]) << (SAINT_BIT - 1));
+ }
+ }
+}
+
+static void libsais_final_bwt_aux_scan_left_to_right_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t rm,
+ sa_sint_t * RESTRICT I, sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 1; i < j; i += 2) {
+ prefetchw(&SA[i + 2 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i + prefetch_distance + 0];
+ const u8 * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ Ts0--;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i + prefetch_distance + 1];
+ const u8 * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ Ts1--;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+
+ sa_sint_t p0 = SA[i + 0];
+ SA[i + 0] = p0 & SAINT_MAX;
+ if (p0 > 0) {
+ p0--;
+ SA[i + 0] = T[p0] | SAINT_MIN;
+ SA[induction_bucket[T[p0]]++] = p0 | ((sa_sint_t)(T[p0 - (p0 > 0)] < T[p0]) << (SAINT_BIT - 1));
+ if ((p0 & rm) == 0) {
+ I[p0 / (rm + 1)] = induction_bucket[T[p0]];
+ }
+ }
+ sa_sint_t p1 = SA[i + 1];
+ SA[i + 1] = p1 & SAINT_MAX;
+ if (p1 > 0) {
+ p1--;
+ SA[i + 1] = T[p1] | SAINT_MIN;
+ SA[induction_bucket[T[p1]]++] = p1 | ((sa_sint_t)(T[p1 - (p1 > 0)] < T[p1]) << (SAINT_BIT - 1));
+ if ((p1 & rm) == 0) {
+ I[p1 / (rm + 1)] = induction_bucket[T[p1]];
+ }
+ }
+ }
+
+ for (j += prefetch_distance + 1; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p & SAINT_MAX;
+ if (p > 0) {
+ p--;
+ SA[i] = T[p] | SAINT_MIN;
+ SA[induction_bucket[T[p]]++] = p | ((sa_sint_t)(T[p - (p > 0)] < T[p]) << (SAINT_BIT - 1));
+ if ((p & rm) == 0) {
+ I[p / (rm + 1)] = induction_bucket[T[p]];
+ }
+ }
+ }
+}
+
+static void libsais_final_sorting_scan_left_to_right_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 1; i < j; i += 2) {
+ prefetchw(&SA[i + 2 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i + prefetch_distance + 0];
+ const u8 * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ Ts0--;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i + prefetch_distance + 1];
+ const u8 * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ Ts1--;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+
+ sa_sint_t p0 = SA[i + 0];
+ SA[i + 0] = p0 ^ SAINT_MIN;
+ if (p0 > 0) {
+ p0--;
+ SA[induction_bucket[T[p0]]++] = p0 | ((sa_sint_t)(T[p0 - (p0 > 0)] < T[p0]) << (SAINT_BIT - 1));
+ }
+ sa_sint_t p1 = SA[i + 1];
+ SA[i + 1] = p1 ^ SAINT_MIN;
+ if (p1 > 0) {
+ p1--;
+ SA[induction_bucket[T[p1]]++] = p1 | ((sa_sint_t)(T[p1 - (p1 > 0)] < T[p1]) << (SAINT_BIT - 1));
+ }
+ }
+
+ for (j += prefetch_distance + 1; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p ^ SAINT_MIN;
+ if (p > 0) {
+ p--;
+ SA[induction_bucket[T[p]]++] = p | ((sa_sint_t)(T[p - (p > 0)] < T[p]) << (SAINT_BIT - 1));
+ }
+ }
+}
+
+static void libsais_final_sorting_scan_left_to_right_32s(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - 2 * prefetch_distance - 1; i < j; i += 2) {
+ prefetchw(&SA[i + 3 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i + 2 * prefetch_distance + 0];
+ const sa_sint_t * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i + 2 * prefetch_distance + 1];
+ const sa_sint_t * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ sa_sint_t s2 = SA[i + 1 * prefetch_distance + 0];
+ if (s2 > 0) {
+ prefetchw(&induction_bucket[T[s2 - 1]]);
+ prefetch(&T[s2] - 2);
+ }
+ sa_sint_t s3 = SA[i + 1 * prefetch_distance + 1];
+ if (s3 > 0) {
+ prefetchw(&induction_bucket[T[s3 - 1]]);
+ prefetch(&T[s3] - 2);
+ }
+
+ sa_sint_t p0 = SA[i + 0];
+ SA[i + 0] = p0 ^ SAINT_MIN;
+ if (p0 > 0) {
+ p0--;
+ SA[induction_bucket[T[p0]]++] = p0 | ((sa_sint_t)(T[p0 - (p0 > 0)] < T[p0]) << (SAINT_BIT - 1));
+ }
+ sa_sint_t p1 = SA[i + 1];
+ SA[i + 1] = p1 ^ SAINT_MIN;
+ if (p1 > 0) {
+ p1--;
+ SA[induction_bucket[T[p1]]++] = p1 | ((sa_sint_t)(T[p1 - (p1 > 0)] < T[p1]) << (SAINT_BIT - 1));
+ }
+ }
+
+ for (j += 2 * prefetch_distance + 1; i < j; i += 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p ^ SAINT_MIN;
+ if (p > 0) {
+ p--;
+ SA[induction_bucket[T[p]]++] = p | ((sa_sint_t)(T[p - (p > 0)] < T[p]) << (SAINT_BIT - 1));
+ }
+ }
+}
+static void libsais_final_bwt_scan_left_to_right_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, fast_sint_t n,
+ sa_sint_t * RESTRICT induction_bucket, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ SA[induction_bucket[T[(sa_sint_t)n - 1]]++] =
+ ((sa_sint_t)n - 1) | ((sa_sint_t)(T[(sa_sint_t)n - 2] < T[(sa_sint_t)n - 1]) << (SAINT_BIT - 1));
+
+ if (threads == 1 || n < 65536) {
+ libsais_final_bwt_scan_left_to_right_8u(T, SA, induction_bucket, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static void libsais_final_bwt_aux_scan_left_to_right_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ fast_sint_t n, sa_sint_t rm, sa_sint_t * RESTRICT I,
+ sa_sint_t * RESTRICT induction_bucket, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ SA[induction_bucket[T[(sa_sint_t)n - 1]]++] =
+ ((sa_sint_t)n - 1) | ((sa_sint_t)(T[(sa_sint_t)n - 2] < T[(sa_sint_t)n - 1]) << (SAINT_BIT - 1));
+
+ if ((((sa_sint_t)n - 1) & rm) == 0) {
+ I[((sa_sint_t)n - 1) / (rm + 1)] = induction_bucket[T[(sa_sint_t)n - 1]];
+ }
+
+ if (threads == 1 || n < 65536) {
+ libsais_final_bwt_aux_scan_left_to_right_8u(T, SA, rm, I, induction_bucket, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static void libsais_final_sorting_scan_left_to_right_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ fast_sint_t n, sa_sint_t * RESTRICT induction_bucket,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ SA[induction_bucket[T[(sa_sint_t)n - 1]]++] =
+ ((sa_sint_t)n - 1) | ((sa_sint_t)(T[(sa_sint_t)n - 2] < T[(sa_sint_t)n - 1]) << (SAINT_BIT - 1));
+
+ if (threads == 1 || n < 65536) {
+ libsais_final_sorting_scan_left_to_right_8u(T, SA, induction_bucket, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static void libsais_final_sorting_scan_left_to_right_32s_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT induction_bucket,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ SA[induction_bucket[T[n - 1]]++] = (n - 1) | ((sa_sint_t)(T[n - 2] < T[n - 1]) << (SAINT_BIT - 1));
+
+ if (threads == 1 || n < 65536) {
+ libsais_final_sorting_scan_left_to_right_32s(T, SA, induction_bucket, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static sa_sint_t libsais_final_bwt_scan_right_to_left_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ sa_sint_t index = -1;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + prefetch_distance + 1; i >= j; i -= 2) {
+ prefetchw(&SA[i - 2 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i - prefetch_distance - 0];
+ const u8 * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ Ts0--;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i - prefetch_distance - 1];
+ const u8 * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ Ts1--;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+
+ sa_sint_t p0 = SA[i - 0];
+ index = (p0 == 0) ? (sa_sint_t)(i - 0) : index;
+ SA[i - 0] = p0 & SAINT_MAX;
+ if (p0 > 0) {
+ p0--;
+ u8 c0 = T[p0 - (p0 > 0)], c1 = T[p0];
+ SA[i - 0] = c1;
+ sa_sint_t t = c0 | SAINT_MIN;
+ SA[--induction_bucket[c1]] = (c0 <= c1) ? p0 : t;
+ }
+
+ sa_sint_t p1 = SA[i - 1];
+ index = (p1 == 0) ? (sa_sint_t)(i - 1) : index;
+ SA[i - 1] = p1 & SAINT_MAX;
+ if (p1 > 0) {
+ p1--;
+ u8 c0 = T[p1 - (p1 > 0)], c1 = T[p1];
+ SA[i - 1] = c1;
+ sa_sint_t t = c0 | SAINT_MIN;
+ SA[--induction_bucket[c1]] = (c0 <= c1) ? p1 : t;
+ }
+ }
+
+ for (j -= prefetch_distance + 1; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ index = (p == 0) ? (sa_sint_t)i : index;
+ SA[i] = p & SAINT_MAX;
+ if (p > 0) {
+ p--;
+ u8 c0 = T[p - (p > 0)], c1 = T[p];
+ SA[i] = c1;
+ sa_sint_t t = c0 | SAINT_MIN;
+ SA[--induction_bucket[c1]] = (c0 <= c1) ? p : t;
+ }
+ }
+
+ return index;
+}
+
+static void libsais_final_bwt_aux_scan_right_to_left_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t rm,
+ sa_sint_t * RESTRICT I, sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + prefetch_distance + 1; i >= j; i -= 2) {
+ prefetchw(&SA[i - 2 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i - prefetch_distance - 0];
+ const u8 * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ Ts0--;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i - prefetch_distance - 1];
+ const u8 * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ Ts1--;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+
+ sa_sint_t p0 = SA[i - 0];
+ SA[i - 0] = p0 & SAINT_MAX;
+ if (p0 > 0) {
+ p0--;
+ u8 c0 = T[p0 - (p0 > 0)], c1 = T[p0];
+ SA[i - 0] = c1;
+ sa_sint_t t = c0 | SAINT_MIN;
+ SA[--induction_bucket[c1]] = (c0 <= c1) ? p0 : t;
+ if ((p0 & rm) == 0) {
+ I[p0 / (rm + 1)] = induction_bucket[T[p0]] + 1;
+ }
+ }
+
+ sa_sint_t p1 = SA[i - 1];
+ SA[i - 1] = p1 & SAINT_MAX;
+ if (p1 > 0) {
+ p1--;
+ u8 c0 = T[p1 - (p1 > 0)], c1 = T[p1];
+ SA[i - 1] = c1;
+ sa_sint_t t = c0 | SAINT_MIN;
+ SA[--induction_bucket[c1]] = (c0 <= c1) ? p1 : t;
+ if ((p1 & rm) == 0) {
+ I[p1 / (rm + 1)] = induction_bucket[T[p1]] + 1;
+ }
+ }
+ }
+
+ for (j -= prefetch_distance + 1; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p & SAINT_MAX;
+ if (p > 0) {
+ p--;
+ u8 c0 = T[p - (p > 0)], c1 = T[p];
+ SA[i] = c1;
+ sa_sint_t t = c0 | SAINT_MIN;
+ SA[--induction_bucket[c1]] = (c0 <= c1) ? p : t;
+ if ((p & rm) == 0) {
+ I[p / (rm + 1)] = induction_bucket[T[p]] + 1;
+ }
+ }
+ }
+}
+
+static void libsais_final_sorting_scan_right_to_left_8u(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + prefetch_distance + 1; i >= j; i -= 2) {
+ prefetchw(&SA[i - 2 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i - prefetch_distance - 0];
+ const u8 * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ Ts0--;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i - prefetch_distance - 1];
+ const u8 * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ Ts1--;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+
+ sa_sint_t p0 = SA[i - 0];
+ SA[i - 0] = p0 & SAINT_MAX;
+ if (p0 > 0) {
+ p0--;
+ SA[--induction_bucket[T[p0]]] = p0 | ((sa_sint_t)(T[p0 - (p0 > 0)] > T[p0]) << (SAINT_BIT - 1));
+ }
+ sa_sint_t p1 = SA[i - 1];
+ SA[i - 1] = p1 & SAINT_MAX;
+ if (p1 > 0) {
+ p1--;
+ SA[--induction_bucket[T[p1]]] = p1 | ((sa_sint_t)(T[p1 - (p1 > 0)] > T[p1]) << (SAINT_BIT - 1));
+ }
+ }
+
+ for (j -= prefetch_distance + 1; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p & SAINT_MAX;
+ if (p > 0) {
+ p--;
+ SA[--induction_bucket[T[p]]] = p | ((sa_sint_t)(T[p - (p > 0)] > T[p]) << (SAINT_BIT - 1));
+ }
+ }
+}
+
+static void libsais_final_sorting_scan_right_to_left_32s(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT induction_bucket,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start + omp_block_size - 1, j = omp_block_start + 2 * prefetch_distance + 1; i >= j; i -= 2) {
+ prefetchw(&SA[i - 3 * prefetch_distance]);
+
+ sa_sint_t s0 = SA[i - 2 * prefetch_distance - 0];
+ const sa_sint_t * Ts0 = &T[s0] - 1;
+ prefetch(s0 > 0 ? Ts0 : NULL);
+ sa_sint_t s1 = SA[i - 2 * prefetch_distance - 1];
+ const sa_sint_t * Ts1 = &T[s1] - 1;
+ prefetch(s1 > 0 ? Ts1 : NULL);
+ sa_sint_t s2 = SA[i - 1 * prefetch_distance - 0];
+ if (s2 > 0) {
+ prefetchw(&induction_bucket[T[s2 - 1]]);
+ prefetch(&T[s2] - 2);
+ }
+ sa_sint_t s3 = SA[i - 1 * prefetch_distance - 1];
+ if (s3 > 0) {
+ prefetchw(&induction_bucket[T[s3 - 1]]);
+ prefetch(&T[s3] - 2);
+ }
+
+ sa_sint_t p0 = SA[i - 0];
+ SA[i - 0] = p0 & SAINT_MAX;
+ if (p0 > 0) {
+ p0--;
+ SA[--induction_bucket[T[p0]]] = p0 | ((sa_sint_t)(T[p0 - (p0 > 0)] > T[p0]) << (SAINT_BIT - 1));
+ }
+ sa_sint_t p1 = SA[i - 1];
+ SA[i - 1] = p1 & SAINT_MAX;
+ if (p1 > 0) {
+ p1--;
+ SA[--induction_bucket[T[p1]]] = p1 | ((sa_sint_t)(T[p1 - (p1 > 0)] > T[p1]) << (SAINT_BIT - 1));
+ }
+ }
+
+ for (j -= 2 * prefetch_distance + 1; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ SA[i] = p & SAINT_MAX;
+ if (p > 0) {
+ p--;
+ SA[--induction_bucket[T[p]]] = p | ((sa_sint_t)(T[p - (p > 0)] > T[p]) << (SAINT_BIT - 1));
+ }
+ }
+}
+static sa_sint_t libsais_final_bwt_scan_right_to_left_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT induction_bucket,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t index = -1;
+
+ if (threads == 1 || n < 65536) {
+ index = libsais_final_bwt_scan_right_to_left_8u(T, SA, induction_bucket, 0, n);
+ }
+ (void)(thread_state);
+ return index;
+}
+
+static void libsais_final_bwt_aux_scan_right_to_left_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t rm, sa_sint_t * RESTRICT I,
+ sa_sint_t * RESTRICT induction_bucket, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (threads == 1 || n < 65536) {
+ libsais_final_bwt_aux_scan_right_to_left_8u(T, SA, rm, I, induction_bucket, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static void libsais_final_sorting_scan_right_to_left_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t * RESTRICT induction_bucket, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (threads == 1 || n < 65536) {
+ libsais_final_sorting_scan_right_to_left_8u(T, SA, induction_bucket, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static void libsais_final_sorting_scan_right_to_left_32s_omp(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t * RESTRICT induction_bucket,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (threads == 1 || n < 65536) {
+ libsais_final_sorting_scan_right_to_left_32s(T, SA, induction_bucket, 0, n);
+ }
+ (void)(thread_state);
+}
+
+static void libsais_clear_lms_suffixes_omp(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t k,
+ sa_sint_t * RESTRICT bucket_start, sa_sint_t * RESTRICT bucket_end,
+ sa_sint_t threads) {
+ fast_sint_t c;
+ (void)(threads);
+ (void)(n);
+
+ for (c = 0; c < k; ++c) {
+ if (bucket_end[c] > bucket_start[c]) {
+ memset(&SA[bucket_start[c]], 0, ((size_t)bucket_end[c] - (size_t)bucket_start[c]) * sizeof(sa_sint_t));
+ }
+ }
+}
+
+static sa_sint_t libsais_induce_final_order_8u_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t bwt, sa_sint_t r, sa_sint_t * RESTRICT I,
+ sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (!bwt) {
+ libsais_final_sorting_scan_left_to_right_8u_omp(T, SA, n, &buckets[6 * ALPHABET_SIZE], threads, thread_state);
+ if (threads > 1 && n >= 65536) {
+ libsais_clear_lms_suffixes_omp(SA, n, ALPHABET_SIZE, &buckets[6 * ALPHABET_SIZE],
+ &buckets[7 * ALPHABET_SIZE], threads);
+ }
+ libsais_final_sorting_scan_right_to_left_8u_omp(T, SA, n, &buckets[7 * ALPHABET_SIZE], threads, thread_state);
+ return 0;
+ } else if (I != NULL) {
+ libsais_final_bwt_aux_scan_left_to_right_8u_omp(T, SA, n, r - 1, I, &buckets[6 * ALPHABET_SIZE], threads,
+ thread_state);
+ if (threads > 1 && n >= 65536) {
+ libsais_clear_lms_suffixes_omp(SA, n, ALPHABET_SIZE, &buckets[6 * ALPHABET_SIZE],
+ &buckets[7 * ALPHABET_SIZE], threads);
+ }
+ libsais_final_bwt_aux_scan_right_to_left_8u_omp(T, SA, n, r - 1, I, &buckets[7 * ALPHABET_SIZE], threads,
+ thread_state);
+ return 0;
+ } else {
+ libsais_final_bwt_scan_left_to_right_8u_omp(T, SA, n, &buckets[6 * ALPHABET_SIZE], threads, thread_state);
+ if (threads > 1 && n >= 65536) {
+ libsais_clear_lms_suffixes_omp(SA, n, ALPHABET_SIZE, &buckets[6 * ALPHABET_SIZE],
+ &buckets[7 * ALPHABET_SIZE], threads);
+ }
+ return libsais_final_bwt_scan_right_to_left_8u_omp(T, SA, n, &buckets[7 * ALPHABET_SIZE], threads,
+ thread_state);
+ }
+}
+
+static void libsais_induce_final_order_32s_6k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ libsais_final_sorting_scan_left_to_right_32s_omp(T, SA, n, &buckets[4 * k], threads, thread_state);
+ libsais_final_sorting_scan_right_to_left_32s_omp(T, SA, n, &buckets[5 * k], threads, thread_state);
+}
+
+static void libsais_induce_final_order_32s_4k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ libsais_final_sorting_scan_left_to_right_32s_omp(T, SA, n, &buckets[2 * k], threads, thread_state);
+ libsais_final_sorting_scan_right_to_left_32s_omp(T, SA, n, &buckets[3 * k], threads, thread_state);
+}
+
+static void libsais_induce_final_order_32s_2k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ libsais_final_sorting_scan_left_to_right_32s_omp(T, SA, n, &buckets[1 * k], threads, thread_state);
+ libsais_final_sorting_scan_right_to_left_32s_omp(T, SA, n, &buckets[0 * k], threads, thread_state);
+}
+
+static void libsais_induce_final_order_32s_1k(const sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t k, sa_sint_t * RESTRICT buckets, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ libsais_count_suffixes_32s(T, n, k, buckets);
+ libsais_initialize_buckets_start_32s_1k(k, buckets);
+ libsais_final_sorting_scan_left_to_right_32s_omp(T, SA, n, buckets, threads, thread_state);
+
+ libsais_count_suffixes_32s(T, n, k, buckets);
+ libsais_initialize_buckets_end_32s_1k(k, buckets);
+ libsais_final_sorting_scan_right_to_left_32s_omp(T, SA, n, buckets, threads, thread_state);
+}
+
+static sa_sint_t libsais_renumber_unique_and_nonunique_lms_suffixes_32s(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t m, sa_sint_t f,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT SAm = &SA[m];
+
+ sa_sint_t i, j;
+ for (i = (sa_sint_t)omp_block_start,
+ j = (sa_sint_t)omp_block_start + (sa_sint_t)omp_block_size - 2 * (sa_sint_t)prefetch_distance - 3;
+ i < j; i += 4) {
+ prefetch(&SA[i + 3 * prefetch_distance]);
+
+ prefetchw(&SAm[((sa_uint_t)SA[i + 2 * prefetch_distance + 0]) >> 1]);
+ prefetchw(&SAm[((sa_uint_t)SA[i + 2 * prefetch_distance + 1]) >> 1]);
+ prefetchw(&SAm[((sa_uint_t)SA[i + 2 * prefetch_distance + 2]) >> 1]);
+ prefetchw(&SAm[((sa_uint_t)SA[i + 2 * prefetch_distance + 3]) >> 1]);
+
+ sa_uint_t q0 = (sa_uint_t)SA[i + prefetch_distance + 0];
+ const sa_sint_t * Tq0 = &T[q0];
+ prefetchw(SAm[q0 >> 1] < 0 ? Tq0 : NULL);
+ sa_uint_t q1 = (sa_uint_t)SA[i + prefetch_distance + 1];
+ const sa_sint_t * Tq1 = &T[q1];
+ prefetchw(SAm[q1 >> 1] < 0 ? Tq1 : NULL);
+ sa_uint_t q2 = (sa_uint_t)SA[i + prefetch_distance + 2];
+ const sa_sint_t * Tq2 = &T[q2];
+ prefetchw(SAm[q2 >> 1] < 0 ? Tq2 : NULL);
+ sa_uint_t q3 = (sa_uint_t)SA[i + prefetch_distance + 3];
+ const sa_sint_t * Tq3 = &T[q3];
+ prefetchw(SAm[q3 >> 1] < 0 ? Tq3 : NULL);
+
+ sa_uint_t p0 = (sa_uint_t)SA[i + 0];
+ sa_sint_t s0 = SAm[p0 >> 1];
+ if (s0 < 0) {
+ T[p0] |= SAINT_MIN;
+ f++;
+ s0 = i + 0 + SAINT_MIN + f;
+ }
+ SAm[p0 >> 1] = s0 - f;
+ sa_uint_t p1 = (sa_uint_t)SA[i + 1];
+ sa_sint_t s1 = SAm[p1 >> 1];
+ if (s1 < 0) {
+ T[p1] |= SAINT_MIN;
+ f++;
+ s1 = i + 1 + SAINT_MIN + f;
+ }
+ SAm[p1 >> 1] = s1 - f;
+ sa_uint_t p2 = (sa_uint_t)SA[i + 2];
+ sa_sint_t s2 = SAm[p2 >> 1];
+ if (s2 < 0) {
+ T[p2] |= SAINT_MIN;
+ f++;
+ s2 = i + 2 + SAINT_MIN + f;
+ }
+ SAm[p2 >> 1] = s2 - f;
+ sa_uint_t p3 = (sa_uint_t)SA[i + 3];
+ sa_sint_t s3 = SAm[p3 >> 1];
+ if (s3 < 0) {
+ T[p3] |= SAINT_MIN;
+ f++;
+ s3 = i + 3 + SAINT_MIN + f;
+ }
+ SAm[p3 >> 1] = s3 - f;
+ }
+
+ for (j += 2 * (sa_sint_t)prefetch_distance + 3; i < j; i += 1) {
+ sa_uint_t p = (sa_uint_t)SA[i];
+ sa_sint_t s = SAm[p >> 1];
+ if (s < 0) {
+ T[p] |= SAINT_MIN;
+ f++;
+ s = i + SAINT_MIN + f;
+ }
+ SAm[p >> 1] = s - f;
+ }
+
+ return f;
+}
+
+static void libsais_compact_unique_and_nonunique_lms_suffixes_32s(sa_sint_t * RESTRICT SA, sa_sint_t m,
+ fast_sint_t * pl, fast_sint_t * pr,
+ fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ sa_sint_t * RESTRICT SAl = &SA[0];
+ sa_sint_t * RESTRICT SAr = &SA[0];
+
+ fast_sint_t i, j, l = *pl - 1, r = *pr - 1;
+ for (i = (fast_sint_t)m + omp_block_start + omp_block_size - 1, j = (fast_sint_t)m + omp_block_start + 3; i >= j;
+ i -= 4) {
+ prefetch(&SA[i - prefetch_distance]);
+
+ sa_sint_t p0 = SA[i - 0];
+ SAl[l] = p0 & SAINT_MAX;
+ l -= p0 < 0;
+ SAr[r] = p0 - 1;
+ r -= p0 > 0;
+ sa_sint_t p1 = SA[i - 1];
+ SAl[l] = p1 & SAINT_MAX;
+ l -= p1 < 0;
+ SAr[r] = p1 - 1;
+ r -= p1 > 0;
+ sa_sint_t p2 = SA[i - 2];
+ SAl[l] = p2 & SAINT_MAX;
+ l -= p2 < 0;
+ SAr[r] = p2 - 1;
+ r -= p2 > 0;
+ sa_sint_t p3 = SA[i - 3];
+ SAl[l] = p3 & SAINT_MAX;
+ l -= p3 < 0;
+ SAr[r] = p3 - 1;
+ r -= p3 > 0;
+ }
+
+ for (j -= 3; i >= j; i -= 1) {
+ sa_sint_t p = SA[i];
+ SAl[l] = p & SAINT_MAX;
+ l -= p < 0;
+ SAr[r] = p - 1;
+ r -= p > 0;
+ }
+
+ *pl = l + 1;
+ *pr = r + 1;
+}
+static sa_sint_t libsais_renumber_unique_and_nonunique_lms_suffixes_32s_omp(
+ sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t m, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t f = 0;
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (m / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : m - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ f = libsais_renumber_unique_and_nonunique_lms_suffixes_32s(T, SA, m, 0, omp_block_start, omp_block_size);
+ }
+ }
+
+ return f;
+}
+
+static void libsais_compact_unique_and_nonunique_lms_suffixes_32s_omp(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m,
+ sa_sint_t fs, sa_sint_t f, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (((fast_sint_t)n >> 1) / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size =
+ omp_thread_num < omp_num_threads - 1 ? omp_block_stride : ((fast_sint_t)n >> 1) - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ fast_sint_t l = m, r = (fast_sint_t)n + (fast_sint_t)fs;
+ libsais_compact_unique_and_nonunique_lms_suffixes_32s(SA, m, &l, &r, omp_block_start, omp_block_size);
+ }
+ }
+
+ memcpy(&SA[(fast_sint_t)n + (fast_sint_t)fs - (fast_sint_t)m], &SA[(fast_sint_t)m - (fast_sint_t)f],
+ (size_t)f * sizeof(sa_sint_t));
+}
+
+static sa_sint_t libsais_compact_lms_suffixes_32s_omp(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t m, sa_sint_t fs, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ sa_sint_t f = libsais_renumber_unique_and_nonunique_lms_suffixes_32s_omp(T, SA, m, threads, thread_state);
+ libsais_compact_unique_and_nonunique_lms_suffixes_32s_omp(SA, n, m, fs, f, threads, thread_state);
+
+ return f;
+}
+
+static void libsais_merge_unique_lms_suffixes_32s(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t m, fast_sint_t l, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ const sa_sint_t * RESTRICT SAnm = &SA[(fast_sint_t)n - (fast_sint_t)m - 1 + l];
+
+ sa_sint_t i, j;
+ fast_sint_t tmp = *SAnm++;
+ for (i = (sa_sint_t)omp_block_start, j = (sa_sint_t)omp_block_start + (sa_sint_t)omp_block_size - 6; i < j;
+ i += 4) {
+ prefetch(&T[i + prefetch_distance]);
+
+ sa_sint_t c0 = T[i + 0];
+ if (c0 < 0) {
+ T[i + 0] = c0 & SAINT_MAX;
+ SA[tmp] = i + 0;
+ i++;
+ tmp = *SAnm++;
+ }
+ sa_sint_t c1 = T[i + 1];
+ if (c1 < 0) {
+ T[i + 1] = c1 & SAINT_MAX;
+ SA[tmp] = i + 1;
+ i++;
+ tmp = *SAnm++;
+ }
+ sa_sint_t c2 = T[i + 2];
+ if (c2 < 0) {
+ T[i + 2] = c2 & SAINT_MAX;
+ SA[tmp] = i + 2;
+ i++;
+ tmp = *SAnm++;
+ }
+ sa_sint_t c3 = T[i + 3];
+ if (c3 < 0) {
+ T[i + 3] = c3 & SAINT_MAX;
+ SA[tmp] = i + 3;
+ i++;
+ tmp = *SAnm++;
+ }
+ }
+
+ for (j += 6; i < j; i += 1) {
+ sa_sint_t c = T[i];
+ if (c < 0) {
+ T[i] = c & SAINT_MAX;
+ SA[tmp] = i;
+ i++;
+ tmp = *SAnm++;
+ }
+ }
+}
+
+static void libsais_merge_nonunique_lms_suffixes_32s(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m, fast_sint_t l,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ const sa_sint_t * RESTRICT SAnm = &SA[(fast_sint_t)n - (fast_sint_t)m - 1 + l];
+
+ fast_sint_t i, j;
+ sa_sint_t tmp = *SAnm++;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - 3; i < j; i += 4) {
+ prefetch(&SA[i + prefetch_distance]);
+
+ if (SA[i + 0] == 0) {
+ SA[i + 0] = tmp;
+ tmp = *SAnm++;
+ }
+ if (SA[i + 1] == 0) {
+ SA[i + 1] = tmp;
+ tmp = *SAnm++;
+ }
+ if (SA[i + 2] == 0) {
+ SA[i + 2] = tmp;
+ tmp = *SAnm++;
+ }
+ if (SA[i + 3] == 0) {
+ SA[i + 3] = tmp;
+ tmp = *SAnm++;
+ }
+ }
+
+ for (j += 3; i < j; i += 1) {
+ if (SA[i] == 0) {
+ SA[i] = tmp;
+ tmp = *SAnm++;
+ }
+ }
+}
+
+static void libsais_merge_unique_lms_suffixes_32s_omp(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t m, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (n / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : n - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ libsais_merge_unique_lms_suffixes_32s(T, SA, n, m, 0, omp_block_start, omp_block_size);
+ }
+ }
+}
+
+static void libsais_merge_nonunique_lms_suffixes_32s_omp(sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t m, sa_sint_t f,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ {
+ (void)(threads);
+ (void)(thread_state);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (m / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : m - omp_block_start;
+
+ if (omp_num_threads == 1) {
+ libsais_merge_nonunique_lms_suffixes_32s(SA, n, m, f, omp_block_start, omp_block_size);
+ }
+ }
+}
+
+static void libsais_merge_compacted_lms_suffixes_32s_omp(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n,
+ sa_sint_t m, sa_sint_t f, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ libsais_merge_unique_lms_suffixes_32s_omp(T, SA, n, m, threads, thread_state);
+ libsais_merge_nonunique_lms_suffixes_32s_omp(SA, n, m, f, threads, thread_state);
+}
+
+static void libsais_reconstruct_compacted_lms_suffixes_32s_2k_omp(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t k, sa_sint_t m, sa_sint_t fs,
+ sa_sint_t f, sa_sint_t * RESTRICT buckets,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (f > 0) {
+ memmove(&SA[n - m - 1], &SA[n + fs - m], (size_t)f * sizeof(sa_sint_t));
+
+ libsais_count_and_gather_compacted_lms_suffixes_32s_2k_omp(T, SA, n, k, buckets, threads, thread_state);
+ libsais_reconstruct_lms_suffixes_omp(SA, n, m - f, threads);
+
+ memcpy(&SA[n - m - 1 + f], &SA[0], ((size_t)m - (size_t)f) * sizeof(sa_sint_t));
+ memset(&SA[0], 0, (size_t)m * sizeof(sa_sint_t));
+
+ libsais_merge_compacted_lms_suffixes_32s_omp(T, SA, n, m, f, threads, thread_state);
+ } else {
+ libsais_count_and_gather_lms_suffixes_32s_2k(T, SA, n, k, buckets, 0, n);
+ libsais_reconstruct_lms_suffixes_omp(SA, n, m, threads);
+ }
+}
+
+static void libsais_reconstruct_compacted_lms_suffixes_32s_1k_omp(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA,
+ sa_sint_t n, sa_sint_t m, sa_sint_t fs, sa_sint_t f,
+ sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ if (f > 0) {
+ memmove(&SA[n - m - 1], &SA[n + fs - m], (size_t)f * sizeof(sa_sint_t));
+
+ libsais_gather_compacted_lms_suffixes_32s(T, SA, n);
+ libsais_reconstruct_lms_suffixes_omp(SA, n, m - f, threads);
+
+ memcpy(&SA[n - m - 1 + f], &SA[0], ((size_t)m - (size_t)f) * sizeof(sa_sint_t));
+ memset(&SA[0], 0, (size_t)m * sizeof(sa_sint_t));
+
+ libsais_merge_compacted_lms_suffixes_32s_omp(T, SA, n, m, f, threads, thread_state);
+ } else {
+ libsais_gather_lms_suffixes_32s(T, SA, n);
+ libsais_reconstruct_lms_suffixes_omp(SA, n, m, threads);
+ }
+}
+
+static sa_sint_t libsais_main_32s(sa_sint_t * RESTRICT T, sa_sint_t * RESTRICT SA, sa_sint_t n, sa_sint_t k,
+ sa_sint_t fs, sa_sint_t threads, LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ fs = fs < (SAINT_MAX - n) ? fs : (SAINT_MAX - n);
+
+ if (k > 0 && fs / k >= 6) {
+ sa_sint_t alignment = (fs - 1024) / k >= 6 ? 1024 : 16;
+ sa_sint_t * RESTRICT buckets =
+ (fs - alignment) / k >= 6
+ ? (sa_sint_t *)libsais_align_up(&SA[n + fs - 6 * k - alignment], (size_t)alignment * sizeof(sa_sint_t))
+ : &SA[n + fs - 6 * k];
+
+ sa_sint_t m = libsais_count_and_gather_lms_suffixes_32s_4k_omp(T, SA, n, k, buckets, threads, thread_state);
+ if (m > 1) {
+ memset(SA, 0, ((size_t)n - (size_t)m) * sizeof(sa_sint_t));
+
+ sa_sint_t first_lms_suffix = SA[n - m];
+ sa_sint_t left_suffixes_count =
+ libsais_initialize_buckets_for_lms_suffixes_radix_sort_32s_6k(T, k, buckets, first_lms_suffix);
+
+ libsais_radix_sort_lms_suffixes_32s_6k_omp(T, SA, n, m, &buckets[4 * k], threads, thread_state);
+ libsais_radix_sort_set_markers_32s_6k_omp(SA, k, &buckets[4 * k], threads);
+
+ if (threads > 1 && n >= 65536) {
+ memset(&SA[(fast_sint_t)n - (fast_sint_t)m], 0, (size_t)m * sizeof(sa_sint_t));
+ }
+
+ libsais_initialize_buckets_for_partial_sorting_32s_6k(T, k, buckets, first_lms_suffix, left_suffixes_count);
+ libsais_induce_partial_order_32s_6k_omp(T, SA, n, k, buckets, first_lms_suffix, left_suffixes_count,
+ threads, thread_state);
+
+ sa_sint_t names =
+ libsais_renumber_and_mark_distinct_lms_suffixes_32s_4k_omp(SA, n, m, threads, thread_state);
+ if (names < m) {
+ sa_sint_t f = libsais_compact_lms_suffixes_32s_omp(T, SA, n, m, fs, threads, thread_state);
+
+ if (libsais_main_32s(SA + n + fs - m + f, SA, m - f, names - f, fs + n - 2 * m + f, threads,
+ thread_state) != 0) {
+ return -2;
+ }
+
+ libsais_reconstruct_compacted_lms_suffixes_32s_2k_omp(T, SA, n, k, m, fs, f, buckets, threads,
+ thread_state);
+ } else {
+ libsais_count_lms_suffixes_32s_2k(T, n, k, buckets);
+ }
+
+ libsais_initialize_buckets_start_and_end_32s_4k(k, buckets);
+ libsais_place_lms_suffixes_histogram_32s_4k(SA, n, k, m, buckets);
+ libsais_induce_final_order_32s_4k(T, SA, n, k, buckets, threads, thread_state);
+ } else {
+ SA[0] = SA[n - 1];
+
+ libsais_initialize_buckets_start_and_end_32s_6k(k, buckets);
+ libsais_place_lms_suffixes_histogram_32s_6k(SA, n, k, m, buckets);
+ libsais_induce_final_order_32s_6k(T, SA, n, k, buckets, threads, thread_state);
+ }
+
+ return 0;
+ } else if (k > 0 && fs / k >= 4) {
+ sa_sint_t alignment = (fs - 1024) / k >= 4 ? 1024 : 16;
+ sa_sint_t * RESTRICT buckets =
+ (fs - alignment) / k >= 4
+ ? (sa_sint_t *)libsais_align_up(&SA[n + fs - 4 * k - alignment], (size_t)alignment * sizeof(sa_sint_t))
+ : &SA[n + fs - 4 * k];
+
+ sa_sint_t m = libsais_count_and_gather_lms_suffixes_32s_2k_omp(T, SA, n, k, buckets, threads, thread_state);
+ if (m > 1) {
+ libsais_initialize_buckets_for_radix_and_partial_sorting_32s_4k(T, k, buckets, SA[n - m]);
+
+ libsais_radix_sort_lms_suffixes_32s_2k_omp(T, SA, n, m, &buckets[1], threads, thread_state);
+ libsais_radix_sort_set_markers_32s_4k_omp(SA, k, &buckets[1], threads);
+
+ libsais_place_lms_suffixes_interval_32s_4k(SA, n, k, m - 1, buckets);
+ libsais_induce_partial_order_32s_4k_omp(T, SA, n, k, buckets, threads, thread_state);
+
+ sa_sint_t names =
+ libsais_renumber_and_mark_distinct_lms_suffixes_32s_4k_omp(SA, n, m, threads, thread_state);
+ if (names < m) {
+ sa_sint_t f = libsais_compact_lms_suffixes_32s_omp(T, SA, n, m, fs, threads, thread_state);
+
+ if (libsais_main_32s(SA + n + fs - m + f, SA, m - f, names - f, fs + n - 2 * m + f, threads,
+ thread_state) != 0) {
+ return -2;
+ }
+
+ libsais_reconstruct_compacted_lms_suffixes_32s_2k_omp(T, SA, n, k, m, fs, f, buckets, threads,
+ thread_state);
+ } else {
+ libsais_count_lms_suffixes_32s_2k(T, n, k, buckets);
+ }
+ } else {
+ SA[0] = SA[n - 1];
+ }
+
+ libsais_initialize_buckets_start_and_end_32s_4k(k, buckets);
+ libsais_place_lms_suffixes_histogram_32s_4k(SA, n, k, m, buckets);
+ libsais_induce_final_order_32s_4k(T, SA, n, k, buckets, threads, thread_state);
+
+ return 0;
+ } else if (k > 0 && fs / k >= 2) {
+ sa_sint_t alignment = (fs - 1024) / k >= 2 ? 1024 : 16;
+ sa_sint_t * RESTRICT buckets =
+ (fs - alignment) / k >= 2
+ ? (sa_sint_t *)libsais_align_up(&SA[n + fs - 2 * k - alignment], (size_t)alignment * sizeof(sa_sint_t))
+ : &SA[n + fs - 2 * k];
+
+ sa_sint_t m = libsais_count_and_gather_lms_suffixes_32s_2k_omp(T, SA, n, k, buckets, threads, thread_state);
+ if (m > 1) {
+ libsais_initialize_buckets_for_lms_suffixes_radix_sort_32s_2k(T, k, buckets, SA[n - m]);
+
+ libsais_radix_sort_lms_suffixes_32s_2k_omp(T, SA, n, m, &buckets[1], threads, thread_state);
+ libsais_place_lms_suffixes_interval_32s_2k(SA, n, k, m - 1, buckets);
+
+ libsais_initialize_buckets_start_and_end_32s_2k(k, buckets);
+ libsais_induce_partial_order_32s_2k_omp(T, SA, n, k, buckets, threads, thread_state);
+
+ sa_sint_t names = libsais_renumber_and_mark_distinct_lms_suffixes_32s_1k_omp(T, SA, n, m, threads);
+ if (names < m) {
+ sa_sint_t f = libsais_compact_lms_suffixes_32s_omp(T, SA, n, m, fs, threads, thread_state);
+
+ if (libsais_main_32s(SA + n + fs - m + f, SA, m - f, names - f, fs + n - 2 * m + f, threads,
+ thread_state) != 0) {
+ return -2;
+ }
+
+ libsais_reconstruct_compacted_lms_suffixes_32s_2k_omp(T, SA, n, k, m, fs, f, buckets, threads,
+ thread_state);
+ } else {
+ libsais_count_lms_suffixes_32s_2k(T, n, k, buckets);
+ }
+ } else {
+ SA[0] = SA[n - 1];
+ }
+
+ libsais_initialize_buckets_end_32s_2k(k, buckets);
+ libsais_place_lms_suffixes_histogram_32s_2k(SA, n, k, m, buckets);
+
+ libsais_initialize_buckets_start_and_end_32s_2k(k, buckets);
+ libsais_induce_final_order_32s_2k(T, SA, n, k, buckets, threads, thread_state);
+
+ return 0;
+ } else {
+ sa_sint_t * buffer =
+ fs < k ? (sa_sint_t *)libsais_alloc_aligned((size_t)k * sizeof(sa_sint_t), 4096) : (sa_sint_t *)NULL;
+
+ sa_sint_t alignment = fs - 1024 >= k ? 1024 : 16;
+ sa_sint_t * RESTRICT buckets =
+ fs - alignment >= k
+ ? (sa_sint_t *)libsais_align_up(&SA[n + fs - k - alignment], (size_t)alignment * sizeof(sa_sint_t))
+ : fs >= k ? &SA[n + fs - k]
+ : buffer;
+
+ if (buckets == NULL) {
+ return -2;
+ }
+
+ memset(SA, 0, (size_t)n * sizeof(sa_sint_t));
+
+ libsais_count_suffixes_32s(T, n, k, buckets);
+ libsais_initialize_buckets_end_32s_1k(k, buckets);
+
+ sa_sint_t m = libsais_radix_sort_lms_suffixes_32s_1k(T, SA, n, buckets);
+ if (m > 1) {
+ libsais_induce_partial_order_32s_1k_omp(T, SA, n, k, buckets, threads, thread_state);
+
+ sa_sint_t names = libsais_renumber_and_mark_distinct_lms_suffixes_32s_1k_omp(T, SA, n, m, threads);
+ if (names < m) {
+ if (buffer != NULL) {
+ libsais_free_aligned(buffer);
+ buckets = NULL;
+ }
+
+ sa_sint_t f = libsais_compact_lms_suffixes_32s_omp(T, SA, n, m, fs, threads, thread_state);
+
+ if (libsais_main_32s(SA + n + fs - m + f, SA, m - f, names - f, fs + n - 2 * m + f, threads,
+ thread_state) != 0) {
+ return -2;
+ }
+
+ libsais_reconstruct_compacted_lms_suffixes_32s_1k_omp(T, SA, n, m, fs, f, threads, thread_state);
+
+ if (buckets == NULL) {
+ buckets = buffer = (sa_sint_t *)libsais_alloc_aligned((size_t)k * sizeof(sa_sint_t), 4096);
+ }
+ if (buckets == NULL) {
+ return -2;
+ }
+ }
+
+ libsais_count_suffixes_32s(T, n, k, buckets);
+ libsais_initialize_buckets_end_32s_1k(k, buckets);
+ libsais_place_lms_suffixes_interval_32s_1k(T, SA, k, m, buckets);
+ }
+
+ libsais_induce_final_order_32s_1k(T, SA, n, k, buckets, threads, thread_state);
+ libsais_free_aligned(buffer);
+
+ return 0;
+ }
+}
+
+static sa_sint_t libsais_main_8u(const u8 * T, sa_sint_t * SA, sa_sint_t n, sa_sint_t * RESTRICT buckets, sa_sint_t bwt,
+ sa_sint_t r, sa_sint_t * RESTRICT I, sa_sint_t fs, sa_sint_t * freq, sa_sint_t threads,
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state) {
+ fs = fs < (SAINT_MAX - n) ? fs : (SAINT_MAX - n);
+
+ sa_sint_t m = libsais_count_and_gather_lms_suffixes_8u_omp(T, SA, n, buckets, threads, thread_state);
+
+ libsais_initialize_buckets_start_and_end_8u(buckets, freq);
+
+ if (m > 0) {
+ sa_sint_t first_lms_suffix = SA[n - m];
+ sa_sint_t left_suffixes_count =
+ libsais_initialize_buckets_for_lms_suffixes_radix_sort_8u(T, buckets, first_lms_suffix);
+
+ if (threads > 1 && n >= 65536) {
+ memset(SA, 0, ((size_t)n - (size_t)m) * sizeof(sa_sint_t));
+ }
+ libsais_radix_sort_lms_suffixes_8u_omp(T, SA, n, m, buckets, threads, thread_state);
+ if (threads > 1 && n >= 65536) {
+ memset(&SA[(fast_sint_t)n - (fast_sint_t)m], 0, (size_t)m * sizeof(sa_sint_t));
+ }
+
+ libsais_initialize_buckets_for_partial_sorting_8u(T, buckets, first_lms_suffix, left_suffixes_count);
+ libsais_induce_partial_order_8u_omp(T, SA, n, buckets, first_lms_suffix, left_suffixes_count, threads,
+ thread_state);
+
+ sa_sint_t names = libsais_renumber_and_gather_lms_suffixes_8u_omp(SA, n, m, fs, threads, thread_state);
+ if (names < m) {
+ if (libsais_main_32s(SA + n + fs - m, SA, m, names, fs + n - 2 * m, threads, thread_state) != 0) {
+ return -2;
+ }
+
+ libsais_gather_lms_suffixes_8u_omp(T, SA, n, threads, thread_state);
+ libsais_reconstruct_lms_suffixes_omp(SA, n, m, threads);
+ }
+
+ libsais_place_lms_suffixes_interval_8u(SA, n, m, buckets);
+ } else {
+ memset(SA, 0, (size_t)n * sizeof(sa_sint_t));
+ }
+
+ return libsais_induce_final_order_8u_omp(T, SA, n, bwt, r, I, buckets, threads, thread_state);
+}
+
+static sa_sint_t libsais_main(const u8 * T, sa_sint_t * SA, sa_sint_t n, sa_sint_t bwt, sa_sint_t r, sa_sint_t * I,
+ sa_sint_t fs, sa_sint_t * freq, sa_sint_t threads) {
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state = threads > 1 ? libsais_alloc_thread_state(threads) : NULL;
+ sa_sint_t * RESTRICT buckets = (sa_sint_t *)libsais_alloc_aligned(8 * ALPHABET_SIZE * sizeof(sa_sint_t), 4096);
+
+ sa_sint_t index = buckets != NULL && (thread_state != NULL || threads == 1)
+ ? libsais_main_8u(T, SA, n, buckets, bwt, r, I, fs, freq, threads, thread_state)
+ : -2;
+
+ libsais_free_aligned(buckets);
+ libsais_free_thread_state(thread_state);
+
+ return index;
+}
+
+static s32 libsais_main_int(sa_sint_t * T, sa_sint_t * SA, sa_sint_t n, sa_sint_t k, sa_sint_t fs, sa_sint_t threads) {
+ LIBSAIS_THREAD_STATE * RESTRICT thread_state = threads > 1 ? libsais_alloc_thread_state(threads) : NULL;
+
+ sa_sint_t index =
+ thread_state != NULL || threads == 1 ? libsais_main_32s(T, SA, n, k, fs, threads, thread_state) : -2;
+
+ libsais_free_thread_state(thread_state);
+
+ return index;
+}
+
+static sa_sint_t libsais_main_ctx(const LIBSAIS_CONTEXT * ctx, const u8 * T, sa_sint_t * SA, sa_sint_t n, sa_sint_t bwt,
+ sa_sint_t r, sa_sint_t * I, sa_sint_t fs, sa_sint_t * freq) {
+ return ctx != NULL && (ctx->buckets != NULL && (ctx->thread_state != NULL || ctx->threads == 1))
+ ? libsais_main_8u(T, SA, n, ctx->buckets, bwt, r, I, fs, freq, (sa_sint_t)ctx->threads,
+ ctx->thread_state)
+ : -2;
+}
+
+static void libsais_bwt_copy_8u(u8 * RESTRICT U, sa_sint_t * RESTRICT A, sa_sint_t n) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = 0, j = (fast_sint_t)n - 7; i < j; i += 8) {
+ prefetch(&A[i + prefetch_distance]);
+
+ U[i + 0] = (u8)A[i + 0];
+ U[i + 1] = (u8)A[i + 1];
+ U[i + 2] = (u8)A[i + 2];
+ U[i + 3] = (u8)A[i + 3];
+ U[i + 4] = (u8)A[i + 4];
+ U[i + 5] = (u8)A[i + 5];
+ U[i + 6] = (u8)A[i + 6];
+ U[i + 7] = (u8)A[i + 7];
+ }
+
+ for (j += 7; i < j; i += 1) {
+ U[i] = (u8)A[i];
+ }
+}
+static void * libsais_create_ctx(void) { return (void *)libsais_create_ctx_main(1); }
+
+static void libsais_free_ctx(void * ctx) { libsais_free_ctx_main((LIBSAIS_CONTEXT *)ctx); }
+
+static s32 libsais(const u8 * T, s32 * SA, s32 n, s32 fs, s32 * freq) {
+ if ((T == NULL) || (SA == NULL) || (n < 0) || (fs < 0)) {
+ return -1;
+ } else if (n < 2) {
+ if (freq != NULL) {
+ memset(freq, 0, ALPHABET_SIZE * sizeof(s32));
+ }
+ if (n == 1) {
+ SA[0] = 0;
+ if (freq != NULL) {
+ freq[T[0]]++;
+ }
+ }
+ return 0;
+ }
+
+ return libsais_main(T, SA, n, 0, 0, NULL, fs, freq, 1);
+}
+
+static s32 libsais_int(s32 * T, s32 * SA, s32 n, s32 k, s32 fs) {
+ if ((T == NULL) || (SA == NULL) || (n < 0) || (fs < 0)) {
+ return -1;
+ } else if (n < 2) {
+ if (n == 1) {
+ SA[0] = 0;
+ }
+ return 0;
+ }
+
+ return libsais_main_int(T, SA, n, k, fs, 1);
+}
+
+static s32 libsais_ctx(const void * ctx, const u8 * T, s32 * SA, s32 n, s32 fs, s32 * freq) {
+ if ((ctx == NULL) || (T == NULL) || (SA == NULL) || (n < 0) || (fs < 0)) {
+ return -1;
+ } else if (n < 2) {
+ if (freq != NULL) {
+ memset(freq, 0, ALPHABET_SIZE * sizeof(s32));
+ }
+ if (n == 1) {
+ SA[0] = 0;
+ if (freq != NULL) {
+ freq[T[0]]++;
+ }
+ }
+ return 0;
+ }
+
+ return libsais_main_ctx((const LIBSAIS_CONTEXT *)ctx, T, SA, n, 0, 0, NULL, fs, freq);
+}
+
+static s32 libsais_bwt(const u8 * T, u8 * U, s32 * A, s32 n, s32 fs, s32 * freq) {
+ if ((T == NULL) || (U == NULL) || (A == NULL) || (n < 0) || (fs < 0)) {
+ return -1;
+ } else if (n <= 1) {
+ if (freq != NULL) {
+ memset(freq, 0, ALPHABET_SIZE * sizeof(s32));
+ }
+ if (n == 1) {
+ U[0] = T[0];
+ if (freq != NULL) {
+ freq[T[0]]++;
+ }
+ }
+ return n;
+ }
+
+ sa_sint_t index = libsais_main(T, A, n, 1, 0, NULL, fs, freq, 1);
+ if (index >= 0) {
+ index++;
+
+ U[0] = T[n - 1];
+ libsais_bwt_copy_8u(U + 1, A, index - 1);
+ libsais_bwt_copy_8u(U + index, A + index, n - index);
+ }
+
+ return index;
+}
+
+static s32 libsais_bwt_aux(const u8 * T, u8 * U, s32 * A, s32 n, s32 fs, s32 * freq, s32 r, s32 * I) {
+ if ((T == NULL) || (U == NULL) || (A == NULL) || (n < 0) || (fs < 0) || (r < 2) || ((r & (r - 1)) != 0) ||
+ (I == NULL)) {
+ return -1;
+ } else if (n <= 1) {
+ if (freq != NULL) {
+ memset(freq, 0, ALPHABET_SIZE * sizeof(s32));
+ }
+ if (n == 1) {
+ U[0] = T[0];
+ if (freq != NULL) {
+ freq[T[0]]++;
+ }
+ }
+ I[0] = n;
+ return 0;
+ }
+
+ if (libsais_main(T, A, n, 1, r, I, fs, freq, 1) != 0) {
+ return -2;
+ }
+
+ U[0] = T[n - 1];
+ libsais_bwt_copy_8u(U + 1, A, I[0] - 1);
+ libsais_bwt_copy_8u(U + I[0], A + I[0], n - I[0]);
+
+ return 0;
+}
+
+static s32 libsais_bwt_ctx(const void * ctx, const u8 * T, u8 * U, s32 * A, s32 n, s32 fs, s32 * freq) {
+ if ((ctx == NULL) || (T == NULL) || (U == NULL) || (A == NULL) || (n < 0) || (fs < 0)) {
+ return -1;
+ } else if (n <= 1) {
+ if (freq != NULL) {
+ memset(freq, 0, ALPHABET_SIZE * sizeof(s32));
+ }
+ if (n == 1) {
+ U[0] = T[0];
+ if (freq != NULL) {
+ freq[T[0]]++;
+ }
+ }
+ return n;
+ }
+
+ sa_sint_t index = libsais_main_ctx((const LIBSAIS_CONTEXT *)ctx, T, A, n, 1, 0, NULL, fs, freq);
+ if (index >= 0) {
+ index++;
+
+ U[0] = T[n - 1];
+
+ libsais_bwt_copy_8u(U + 1, A, index - 1);
+ libsais_bwt_copy_8u(U + index, A + index, n - index);
+ }
+
+ return index;
+}
+
+static s32 libsais_bwt_aux_ctx(const void * ctx, const u8 * T, u8 * U, s32 * A, s32 n, s32 fs, s32 * freq, s32 r,
+ s32 * I) {
+ if ((ctx == NULL) || (T == NULL) || (U == NULL) || (A == NULL) || (n < 0) || (fs < 0) || (r < 2) ||
+ ((r & (r - 1)) != 0) || (I == NULL)) {
+ return -1;
+ } else if (n <= 1) {
+ if (freq != NULL) {
+ memset(freq, 0, ALPHABET_SIZE * sizeof(s32));
+ }
+ if (n == 1) {
+ U[0] = T[0];
+ if (freq != NULL) {
+ freq[T[0]]++;
+ }
+ }
+ I[0] = n;
+ return 0;
+ }
+
+ if (libsais_main_ctx((const LIBSAIS_CONTEXT *)ctx, T, A, n, 1, r, I, fs, freq) != 0) {
+ return -2;
+ }
+
+ U[0] = T[n - 1];
+ libsais_bwt_copy_8u(U + 1, A, I[0] - 1);
+ libsais_bwt_copy_8u(U + I[0], A + I[0], n - I[0]);
+ return 0;
+}
+static LIBSAIS_UNBWT_CONTEXT * libsais_unbwt_create_ctx_main(sa_sint_t threads) {
+ LIBSAIS_UNBWT_CONTEXT * RESTRICT ctx =
+ (LIBSAIS_UNBWT_CONTEXT *)libsais_alloc_aligned(sizeof(LIBSAIS_UNBWT_CONTEXT), 64);
+ sa_uint_t * RESTRICT bucket2 =
+ (sa_uint_t *)libsais_alloc_aligned(ALPHABET_SIZE * ALPHABET_SIZE * sizeof(sa_uint_t), 4096);
+ u16 * RESTRICT fastbits = (u16 *)libsais_alloc_aligned((1 + (1 << UNBWT_FASTBITS)) * sizeof(u16), 4096);
+ sa_uint_t * RESTRICT buckets =
+ threads > 1 ? (sa_uint_t *)libsais_alloc_aligned(
+ (size_t)threads * (ALPHABET_SIZE + (ALPHABET_SIZE * ALPHABET_SIZE)) * sizeof(sa_uint_t), 4096)
+ : NULL;
+
+ if (ctx != NULL && bucket2 != NULL && fastbits != NULL && (buckets != NULL || threads == 1)) {
+ ctx->bucket2 = bucket2;
+ ctx->fastbits = fastbits;
+ ctx->buckets = buckets;
+ ctx->threads = threads;
+
+ return ctx;
+ }
+
+ libsais_free_aligned(buckets);
+ libsais_free_aligned(fastbits);
+ libsais_free_aligned(bucket2);
+ libsais_free_aligned(ctx);
+
+ return NULL;
+}
+
+static void libsais_unbwt_free_ctx_main(LIBSAIS_UNBWT_CONTEXT * ctx) {
+ if (ctx != NULL) {
+ libsais_free_aligned(ctx->buckets);
+ libsais_free_aligned(ctx->fastbits);
+ libsais_free_aligned(ctx->bucket2);
+ libsais_free_aligned(ctx);
+ }
+}
+
+static void libsais_unbwt_compute_histogram(const u8 * RESTRICT T, fast_sint_t n, sa_uint_t * RESTRICT count) {
+ const fast_sint_t prefetch_distance = 256;
+
+ const u8 * RESTRICT T_p = T;
+
+ if (n >= 1024) {
+ sa_uint_t copy[4 * (ALPHABET_SIZE + 16)];
+
+ memset(copy, 0, 4 * (ALPHABET_SIZE + 16) * sizeof(sa_uint_t));
+
+ sa_uint_t * RESTRICT copy0 = copy + 0 * (ALPHABET_SIZE + 16);
+ sa_uint_t * RESTRICT copy1 = copy + 1 * (ALPHABET_SIZE + 16);
+ sa_uint_t * RESTRICT copy2 = copy + 2 * (ALPHABET_SIZE + 16);
+ sa_uint_t * RESTRICT copy3 = copy + 3 * (ALPHABET_SIZE + 16);
+
+ for (; T_p < (u8 *)((ptrdiff_t)(T + 63) & (-64)); T_p += 1) {
+ copy0[T_p[0]]++;
+ }
+
+ fast_uint_t x = ((const u32 *)(const void *)T_p)[0], y = ((const u32 *)(const void *)T_p)[1];
+
+ for (; T_p < (u8 *)((ptrdiff_t)(T + n - 8) & (-64)); T_p += 64) {
+ prefetch(&T_p[prefetch_distance]);
+
+ fast_uint_t z = ((const u32 *)(const void *)T_p)[2], w = ((const u32 *)(const void *)T_p)[3];
+ copy0[(u8)x]++;
+ x >>= 8;
+ copy1[(u8)x]++;
+ x >>= 8;
+ copy2[(u8)x]++;
+ x >>= 8;
+ copy3[x]++;
+ copy0[(u8)y]++;
+ y >>= 8;
+ copy1[(u8)y]++;
+ y >>= 8;
+ copy2[(u8)y]++;
+ y >>= 8;
+ copy3[y]++;
+
+ x = ((const u32 *)(const void *)T_p)[4];
+ y = ((const u32 *)(const void *)T_p)[5];
+ copy0[(u8)z]++;
+ z >>= 8;
+ copy1[(u8)z]++;
+ z >>= 8;
+ copy2[(u8)z]++;
+ z >>= 8;
+ copy3[z]++;
+ copy0[(u8)w]++;
+ w >>= 8;
+ copy1[(u8)w]++;
+ w >>= 8;
+ copy2[(u8)w]++;
+ w >>= 8;
+ copy3[w]++;
+
+ z = ((const u32 *)(const void *)T_p)[6];
+ w = ((const u32 *)(const void *)T_p)[7];
+ copy0[(u8)x]++;
+ x >>= 8;
+ copy1[(u8)x]++;
+ x >>= 8;
+ copy2[(u8)x]++;
+ x >>= 8;
+ copy3[x]++;
+ copy0[(u8)y]++;
+ y >>= 8;
+ copy1[(u8)y]++;
+ y >>= 8;
+ copy2[(u8)y]++;
+ y >>= 8;
+ copy3[y]++;
+
+ x = ((const u32 *)(const void *)T_p)[8];
+ y = ((const u32 *)(const void *)T_p)[9];
+ copy0[(u8)z]++;
+ z >>= 8;
+ copy1[(u8)z]++;
+ z >>= 8;
+ copy2[(u8)z]++;
+ z >>= 8;
+ copy3[z]++;
+ copy0[(u8)w]++;
+ w >>= 8;
+ copy1[(u8)w]++;
+ w >>= 8;
+ copy2[(u8)w]++;
+ w >>= 8;
+ copy3[w]++;
+
+ z = ((const u32 *)(const void *)T_p)[10];
+ w = ((const u32 *)(const void *)T_p)[11];
+ copy0[(u8)x]++;
+ x >>= 8;
+ copy1[(u8)x]++;
+ x >>= 8;
+ copy2[(u8)x]++;
+ x >>= 8;
+ copy3[x]++;
+ copy0[(u8)y]++;
+ y >>= 8;
+ copy1[(u8)y]++;
+ y >>= 8;
+ copy2[(u8)y]++;
+ y >>= 8;
+ copy3[y]++;
+
+ x = ((const u32 *)(const void *)T_p)[12];
+ y = ((const u32 *)(const void *)T_p)[13];
+ copy0[(u8)z]++;
+ z >>= 8;
+ copy1[(u8)z]++;
+ z >>= 8;
+ copy2[(u8)z]++;
+ z >>= 8;
+ copy3[z]++;
+ copy0[(u8)w]++;
+ w >>= 8;
+ copy1[(u8)w]++;
+ w >>= 8;
+ copy2[(u8)w]++;
+ w >>= 8;
+ copy3[w]++;
+
+ z = ((const u32 *)(const void *)T_p)[14];
+ w = ((const u32 *)(const void *)T_p)[15];
+ copy0[(u8)x]++;
+ x >>= 8;
+ copy1[(u8)x]++;
+ x >>= 8;
+ copy2[(u8)x]++;
+ x >>= 8;
+ copy3[x]++;
+ copy0[(u8)y]++;
+ y >>= 8;
+ copy1[(u8)y]++;
+ y >>= 8;
+ copy2[(u8)y]++;
+ y >>= 8;
+ copy3[y]++;
+
+ x = ((const u32 *)(const void *)T_p)[16];
+ y = ((const u32 *)(const void *)T_p)[17];
+ copy0[(u8)z]++;
+ z >>= 8;
+ copy1[(u8)z]++;
+ z >>= 8;
+ copy2[(u8)z]++;
+ z >>= 8;
+ copy3[z]++;
+ copy0[(u8)w]++;
+ w >>= 8;
+ copy1[(u8)w]++;
+ w >>= 8;
+ copy2[(u8)w]++;
+ w >>= 8;
+ copy3[w]++;
+ }
+
+ copy0[(u8)x]++;
+ x >>= 8;
+ copy1[(u8)x]++;
+ x >>= 8;
+ copy2[(u8)x]++;
+ x >>= 8;
+ copy3[x]++;
+ copy0[(u8)y]++;
+ y >>= 8;
+ copy1[(u8)y]++;
+ y >>= 8;
+ copy2[(u8)y]++;
+ y >>= 8;
+ copy3[y]++;
+
+ T_p += 8;
+
+ fast_uint_t i;
+ for (i = 0; i < ALPHABET_SIZE; i++) {
+ count[i] += copy0[i] + copy1[i] + copy2[i] + copy3[i];
+ }
+ }
+
+ for (; T_p < T + n; T_p += 1) {
+ count[T_p[0]]++;
+ }
+}
+
+static void libsais_unbwt_transpose_bucket2(sa_uint_t * RESTRICT bucket2) {
+ fast_uint_t x, y, c, d;
+ for (x = 0; x != ALPHABET_SIZE; x += 16) {
+ for (c = x; c != x + 16; ++c) {
+ for (d = c + 1; d != x + 16; ++d) {
+ sa_uint_t tmp = bucket2[(d << 8) + c];
+ bucket2[(d << 8) + c] = bucket2[(c << 8) + d];
+ bucket2[(c << 8) + d] = tmp;
+ }
+ }
+
+ for (y = x + 16; y != ALPHABET_SIZE; y += 16) {
+ for (c = x; c != x + 16; ++c) {
+ sa_uint_t * bucket2_yc = &bucket2[(y << 8) + c];
+ sa_uint_t * bucket2_cy = &bucket2[(c << 8) + y];
+
+ sa_uint_t tmp00 = bucket2_yc[0 * 256];
+ bucket2_yc[0 * 256] = bucket2_cy[0];
+ bucket2_cy[0] = tmp00;
+ sa_uint_t tmp01 = bucket2_yc[1 * 256];
+ bucket2_yc[1 * 256] = bucket2_cy[1];
+ bucket2_cy[1] = tmp01;
+ sa_uint_t tmp02 = bucket2_yc[2 * 256];
+ bucket2_yc[2 * 256] = bucket2_cy[2];
+ bucket2_cy[2] = tmp02;
+ sa_uint_t tmp03 = bucket2_yc[3 * 256];
+ bucket2_yc[3 * 256] = bucket2_cy[3];
+ bucket2_cy[3] = tmp03;
+ sa_uint_t tmp04 = bucket2_yc[4 * 256];
+ bucket2_yc[4 * 256] = bucket2_cy[4];
+ bucket2_cy[4] = tmp04;
+ sa_uint_t tmp05 = bucket2_yc[5 * 256];
+ bucket2_yc[5 * 256] = bucket2_cy[5];
+ bucket2_cy[5] = tmp05;
+ sa_uint_t tmp06 = bucket2_yc[6 * 256];
+ bucket2_yc[6 * 256] = bucket2_cy[6];
+ bucket2_cy[6] = tmp06;
+ sa_uint_t tmp07 = bucket2_yc[7 * 256];
+ bucket2_yc[7 * 256] = bucket2_cy[7];
+ bucket2_cy[7] = tmp07;
+ sa_uint_t tmp08 = bucket2_yc[8 * 256];
+ bucket2_yc[8 * 256] = bucket2_cy[8];
+ bucket2_cy[8] = tmp08;
+ sa_uint_t tmp09 = bucket2_yc[9 * 256];
+ bucket2_yc[9 * 256] = bucket2_cy[9];
+ bucket2_cy[9] = tmp09;
+ sa_uint_t tmp10 = bucket2_yc[10 * 256];
+ bucket2_yc[10 * 256] = bucket2_cy[10];
+ bucket2_cy[10] = tmp10;
+ sa_uint_t tmp11 = bucket2_yc[11 * 256];
+ bucket2_yc[11 * 256] = bucket2_cy[11];
+ bucket2_cy[11] = tmp11;
+ sa_uint_t tmp12 = bucket2_yc[12 * 256];
+ bucket2_yc[12 * 256] = bucket2_cy[12];
+ bucket2_cy[12] = tmp12;
+ sa_uint_t tmp13 = bucket2_yc[13 * 256];
+ bucket2_yc[13 * 256] = bucket2_cy[13];
+ bucket2_cy[13] = tmp13;
+ sa_uint_t tmp14 = bucket2_yc[14 * 256];
+ bucket2_yc[14 * 256] = bucket2_cy[14];
+ bucket2_cy[14] = tmp14;
+ sa_uint_t tmp15 = bucket2_yc[15 * 256];
+ bucket2_yc[15 * 256] = bucket2_cy[15];
+ bucket2_cy[15] = tmp15;
+ }
+ }
+ }
+}
+
+static void libsais_unbwt_compute_bigram_histogram_single(const u8 * RESTRICT T, sa_uint_t * RESTRICT bucket1,
+ sa_uint_t * RESTRICT bucket2, fast_uint_t index) {
+ fast_uint_t sum, c;
+ for (sum = 1, c = 0; c < ALPHABET_SIZE; ++c) {
+ fast_uint_t prev = sum;
+ sum += bucket1[c];
+ bucket1[c] = (sa_uint_t)prev;
+ if (prev != sum) {
+ sa_uint_t * RESTRICT bucket2_p = &bucket2[c << 8];
+
+ {
+ fast_uint_t hi = index;
+ if (sum < hi) {
+ hi = sum;
+ }
+ libsais_unbwt_compute_histogram(&T[prev], (fast_sint_t)(hi - prev), bucket2_p);
+ }
+
+ {
+ fast_uint_t lo = index + 1;
+ if (prev > lo) {
+ lo = prev;
+ }
+ libsais_unbwt_compute_histogram(&T[lo - 1], (fast_sint_t)(sum - lo), bucket2_p);
+ }
+ }
+ }
+
+ libsais_unbwt_transpose_bucket2(bucket2);
+}
+
+static void libsais_unbwt_calculate_fastbits(sa_uint_t * RESTRICT bucket2, u16 * RESTRICT fastbits, fast_uint_t lastc,
+ fast_uint_t shift) {
+ fast_uint_t v, w, sum, c, d;
+ for (v = 0, w = 0, sum = 1, c = 0; c < ALPHABET_SIZE; ++c) {
+ if (c == lastc) {
+ sum += 1;
+ }
+
+ for (d = 0; d < ALPHABET_SIZE; ++d, ++w) {
+ fast_uint_t prev = sum;
+ sum += bucket2[w];
+ bucket2[w] = (sa_uint_t)prev;
+ if (prev != sum) {
+ for (; v <= ((sum - 1) >> shift); ++v) {
+ fastbits[v] = (u16)w;
+ }
+ }
+ }
+ }
+}
+
+static void libsais_unbwt_calculate_biPSI(const u8 * RESTRICT T, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket1,
+ sa_uint_t * RESTRICT bucket2, fast_uint_t index, fast_sint_t omp_block_start,
+ fast_sint_t omp_block_end) {
+ {
+ fast_sint_t i = omp_block_start, j = (fast_sint_t)index;
+ if (omp_block_end < j) {
+ j = omp_block_end;
+ }
+ for (; i < j; ++i) {
+ fast_uint_t c = T[i];
+ fast_uint_t p = bucket1[c]++;
+ fast_sint_t t = (fast_sint_t)(index - p);
+
+ if (t != 0) {
+ fast_uint_t w = (((fast_uint_t)T[p + (fast_uint_t)(t >> ((sizeof(fast_sint_t) * 8) - 1))]) << 8) + c;
+ P[bucket2[w]++] = (sa_uint_t)i;
+ }
+ }
+ }
+
+ {
+ fast_sint_t i = (fast_sint_t)index, j = omp_block_end;
+ if (omp_block_start > i) {
+ i = omp_block_start;
+ }
+ for (i += 1; i <= j; ++i) {
+ fast_uint_t c = T[i - 1];
+ fast_uint_t p = bucket1[c]++;
+ fast_sint_t t = (fast_sint_t)(index - p);
+
+ if (t != 0) {
+ fast_uint_t w = (((fast_uint_t)T[p + (fast_uint_t)(t >> ((sizeof(fast_sint_t) * 8) - 1))]) << 8) + c;
+ P[bucket2[w]++] = (sa_uint_t)i;
+ }
+ }
+ }
+}
+
+static void libsais_unbwt_init_single(const u8 * RESTRICT T, sa_uint_t * RESTRICT P, sa_sint_t n,
+ const sa_sint_t * freq, const sa_uint_t * RESTRICT I,
+ sa_uint_t * RESTRICT bucket2, u16 * RESTRICT fastbits) {
+ sa_uint_t bucket1[ALPHABET_SIZE];
+
+ fast_uint_t index = I[0];
+ fast_uint_t lastc = T[0];
+ fast_uint_t shift = 0;
+ while ((n >> shift) > (1 << UNBWT_FASTBITS)) {
+ shift++;
+ }
+
+ if (freq != NULL) {
+ memcpy(bucket1, freq, ALPHABET_SIZE * sizeof(sa_uint_t));
+ } else {
+ memset(bucket1, 0, ALPHABET_SIZE * sizeof(sa_uint_t));
+ libsais_unbwt_compute_histogram(T, n, bucket1);
+ }
+
+ memset(bucket2, 0, ALPHABET_SIZE * ALPHABET_SIZE * sizeof(sa_uint_t));
+ libsais_unbwt_compute_bigram_histogram_single(T, bucket1, bucket2, index);
+
+ libsais_unbwt_calculate_fastbits(bucket2, fastbits, lastc, shift);
+ libsais_unbwt_calculate_biPSI(T, P, bucket1, bucket2, index, 0, n);
+}
+static void libsais_unbwt_decode_1(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, fast_uint_t shift, fast_uint_t * i0, fast_uint_t k) {
+ u16 * RESTRICT U0 = (u16 *)(void *)U;
+
+ fast_uint_t i, p0 = *i0;
+
+ for (i = 0; i != k; ++i) {
+ u16 c0 = fastbits[p0 >> shift];
+ if (bucket2[c0] <= p0) {
+ do {
+ c0++;
+ } while (bucket2[c0] <= p0);
+ }
+ p0 = P[p0];
+ U0[i] = bswap16(c0);
+ }
+
+ *i0 = p0;
+}
+
+static void libsais_unbwt_decode_2(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, fast_uint_t shift, fast_uint_t r, fast_uint_t * i0,
+ fast_uint_t * i1, fast_uint_t k) {
+ u16 * RESTRICT U0 = (u16 *)(void *)U;
+ u16 * RESTRICT U1 = (u16 *)(void *)(((u8 *)U0) + r);
+
+ fast_uint_t i, p0 = *i0, p1 = *i1;
+
+ for (i = 0; i != k; ++i) {
+ u16 c0 = fastbits[p0 >> shift];
+ if (bucket2[c0] <= p0) {
+ do {
+ c0++;
+ } while (bucket2[c0] <= p0);
+ }
+ p0 = P[p0];
+ U0[i] = bswap16(c0);
+ u16 c1 = fastbits[p1 >> shift];
+ if (bucket2[c1] <= p1) {
+ do {
+ c1++;
+ } while (bucket2[c1] <= p1);
+ }
+ p1 = P[p1];
+ U1[i] = bswap16(c1);
+ }
+
+ *i0 = p0;
+ *i1 = p1;
+}
+
+static void libsais_unbwt_decode_3(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, fast_uint_t shift, fast_uint_t r, fast_uint_t * i0,
+ fast_uint_t * i1, fast_uint_t * i2, fast_uint_t k) {
+ u16 * RESTRICT U0 = (u16 *)(void *)U;
+ u16 * RESTRICT U1 = (u16 *)(void *)(((u8 *)U0) + r);
+ u16 * RESTRICT U2 = (u16 *)(void *)(((u8 *)U1) + r);
+
+ fast_uint_t i, p0 = *i0, p1 = *i1, p2 = *i2;
+
+ for (i = 0; i != k; ++i) {
+ u16 c0 = fastbits[p0 >> shift];
+ if (bucket2[c0] <= p0) {
+ do {
+ c0++;
+ } while (bucket2[c0] <= p0);
+ }
+ p0 = P[p0];
+ U0[i] = bswap16(c0);
+ u16 c1 = fastbits[p1 >> shift];
+ if (bucket2[c1] <= p1) {
+ do {
+ c1++;
+ } while (bucket2[c1] <= p1);
+ }
+ p1 = P[p1];
+ U1[i] = bswap16(c1);
+ u16 c2 = fastbits[p2 >> shift];
+ if (bucket2[c2] <= p2) {
+ do {
+ c2++;
+ } while (bucket2[c2] <= p2);
+ }
+ p2 = P[p2];
+ U2[i] = bswap16(c2);
+ }
+
+ *i0 = p0;
+ *i1 = p1;
+ *i2 = p2;
+}
+
+static void libsais_unbwt_decode_4(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, fast_uint_t shift, fast_uint_t r, fast_uint_t * i0,
+ fast_uint_t * i1, fast_uint_t * i2, fast_uint_t * i3, fast_uint_t k) {
+ u16 * RESTRICT U0 = (u16 *)(void *)U;
+ u16 * RESTRICT U1 = (u16 *)(void *)(((u8 *)U0) + r);
+ u16 * RESTRICT U2 = (u16 *)(void *)(((u8 *)U1) + r);
+ u16 * RESTRICT U3 = (u16 *)(void *)(((u8 *)U2) + r);
+
+ fast_uint_t i, p0 = *i0, p1 = *i1, p2 = *i2, p3 = *i3;
+
+ for (i = 0; i != k; ++i) {
+ u16 c0 = fastbits[p0 >> shift];
+ if (bucket2[c0] <= p0) {
+ do {
+ c0++;
+ } while (bucket2[c0] <= p0);
+ }
+ p0 = P[p0];
+ U0[i] = bswap16(c0);
+ u16 c1 = fastbits[p1 >> shift];
+ if (bucket2[c1] <= p1) {
+ do {
+ c1++;
+ } while (bucket2[c1] <= p1);
+ }
+ p1 = P[p1];
+ U1[i] = bswap16(c1);
+ u16 c2 = fastbits[p2 >> shift];
+ if (bucket2[c2] <= p2) {
+ do {
+ c2++;
+ } while (bucket2[c2] <= p2);
+ }
+ p2 = P[p2];
+ U2[i] = bswap16(c2);
+ u16 c3 = fastbits[p3 >> shift];
+ if (bucket2[c3] <= p3) {
+ do {
+ c3++;
+ } while (bucket2[c3] <= p3);
+ }
+ p3 = P[p3];
+ U3[i] = bswap16(c3);
+ }
+
+ *i0 = p0;
+ *i1 = p1;
+ *i2 = p2;
+ *i3 = p3;
+}
+
+static void libsais_unbwt_decode_5(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, fast_uint_t shift, fast_uint_t r, fast_uint_t * i0,
+ fast_uint_t * i1, fast_uint_t * i2, fast_uint_t * i3, fast_uint_t * i4,
+ fast_uint_t k) {
+ u16 * RESTRICT U0 = (u16 *)(void *)U;
+ u16 * RESTRICT U1 = (u16 *)(void *)(((u8 *)U0) + r);
+ u16 * RESTRICT U2 = (u16 *)(void *)(((u8 *)U1) + r);
+ u16 * RESTRICT U3 = (u16 *)(void *)(((u8 *)U2) + r);
+ u16 * RESTRICT U4 = (u16 *)(void *)(((u8 *)U3) + r);
+
+ fast_uint_t i, p0 = *i0, p1 = *i1, p2 = *i2, p3 = *i3, p4 = *i4;
+
+ for (i = 0; i != k; ++i) {
+ u16 c0 = fastbits[p0 >> shift];
+ if (bucket2[c0] <= p0) {
+ do {
+ c0++;
+ } while (bucket2[c0] <= p0);
+ }
+ p0 = P[p0];
+ U0[i] = bswap16(c0);
+ u16 c1 = fastbits[p1 >> shift];
+ if (bucket2[c1] <= p1) {
+ do {
+ c1++;
+ } while (bucket2[c1] <= p1);
+ }
+ p1 = P[p1];
+ U1[i] = bswap16(c1);
+ u16 c2 = fastbits[p2 >> shift];
+ if (bucket2[c2] <= p2) {
+ do {
+ c2++;
+ } while (bucket2[c2] <= p2);
+ }
+ p2 = P[p2];
+ U2[i] = bswap16(c2);
+ u16 c3 = fastbits[p3 >> shift];
+ if (bucket2[c3] <= p3) {
+ do {
+ c3++;
+ } while (bucket2[c3] <= p3);
+ }
+ p3 = P[p3];
+ U3[i] = bswap16(c3);
+ u16 c4 = fastbits[p4 >> shift];
+ if (bucket2[c4] <= p4) {
+ do {
+ c4++;
+ } while (bucket2[c4] <= p4);
+ }
+ p4 = P[p4];
+ U4[i] = bswap16(c4);
+ }
+
+ *i0 = p0;
+ *i1 = p1;
+ *i2 = p2;
+ *i3 = p3;
+ *i4 = p4;
+}
+
+static void libsais_unbwt_decode_6(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, fast_uint_t shift, fast_uint_t r, fast_uint_t * i0,
+ fast_uint_t * i1, fast_uint_t * i2, fast_uint_t * i3, fast_uint_t * i4,
+ fast_uint_t * i5, fast_uint_t k) {
+ u16 * RESTRICT U0 = (u16 *)(void *)U;
+ u16 * RESTRICT U1 = (u16 *)(void *)(((u8 *)U0) + r);
+ u16 * RESTRICT U2 = (u16 *)(void *)(((u8 *)U1) + r);
+ u16 * RESTRICT U3 = (u16 *)(void *)(((u8 *)U2) + r);
+ u16 * RESTRICT U4 = (u16 *)(void *)(((u8 *)U3) + r);
+ u16 * RESTRICT U5 = (u16 *)(void *)(((u8 *)U4) + r);
+
+ fast_uint_t i, p0 = *i0, p1 = *i1, p2 = *i2, p3 = *i3, p4 = *i4, p5 = *i5;
+
+ for (i = 0; i != k; ++i) {
+ u16 c0 = fastbits[p0 >> shift];
+ if (bucket2[c0] <= p0) {
+ do {
+ c0++;
+ } while (bucket2[c0] <= p0);
+ }
+ p0 = P[p0];
+ U0[i] = bswap16(c0);
+ u16 c1 = fastbits[p1 >> shift];
+ if (bucket2[c1] <= p1) {
+ do {
+ c1++;
+ } while (bucket2[c1] <= p1);
+ }
+ p1 = P[p1];
+ U1[i] = bswap16(c1);
+ u16 c2 = fastbits[p2 >> shift];
+ if (bucket2[c2] <= p2) {
+ do {
+ c2++;
+ } while (bucket2[c2] <= p2);
+ }
+ p2 = P[p2];
+ U2[i] = bswap16(c2);
+ u16 c3 = fastbits[p3 >> shift];
+ if (bucket2[c3] <= p3) {
+ do {
+ c3++;
+ } while (bucket2[c3] <= p3);
+ }
+ p3 = P[p3];
+ U3[i] = bswap16(c3);
+ u16 c4 = fastbits[p4 >> shift];
+ if (bucket2[c4] <= p4) {
+ do {
+ c4++;
+ } while (bucket2[c4] <= p4);
+ }
+ p4 = P[p4];
+ U4[i] = bswap16(c4);
+ u16 c5 = fastbits[p5 >> shift];
+ if (bucket2[c5] <= p5) {
+ do {
+ c5++;
+ } while (bucket2[c5] <= p5);
+ }
+ p5 = P[p5];
+ U5[i] = bswap16(c5);
+ }
+
+ *i0 = p0;
+ *i1 = p1;
+ *i2 = p2;
+ *i3 = p3;
+ *i4 = p4;
+ *i5 = p5;
+}
+
+static void libsais_unbwt_decode_7(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, fast_uint_t shift, fast_uint_t r, fast_uint_t * i0,
+ fast_uint_t * i1, fast_uint_t * i2, fast_uint_t * i3, fast_uint_t * i4,
+ fast_uint_t * i5, fast_uint_t * i6, fast_uint_t k) {
+ u16 * RESTRICT U0 = (u16 *)(void *)U;
+ u16 * RESTRICT U1 = (u16 *)(void *)(((u8 *)U0) + r);
+ u16 * RESTRICT U2 = (u16 *)(void *)(((u8 *)U1) + r);
+ u16 * RESTRICT U3 = (u16 *)(void *)(((u8 *)U2) + r);
+ u16 * RESTRICT U4 = (u16 *)(void *)(((u8 *)U3) + r);
+ u16 * RESTRICT U5 = (u16 *)(void *)(((u8 *)U4) + r);
+ u16 * RESTRICT U6 = (u16 *)(void *)(((u8 *)U5) + r);
+
+ fast_uint_t i, p0 = *i0, p1 = *i1, p2 = *i2, p3 = *i3, p4 = *i4, p5 = *i5, p6 = *i6;
+
+ for (i = 0; i != k; ++i) {
+ u16 c0 = fastbits[p0 >> shift];
+ if (bucket2[c0] <= p0) {
+ do {
+ c0++;
+ } while (bucket2[c0] <= p0);
+ }
+ p0 = P[p0];
+ U0[i] = bswap16(c0);
+ u16 c1 = fastbits[p1 >> shift];
+ if (bucket2[c1] <= p1) {
+ do {
+ c1++;
+ } while (bucket2[c1] <= p1);
+ }
+ p1 = P[p1];
+ U1[i] = bswap16(c1);
+ u16 c2 = fastbits[p2 >> shift];
+ if (bucket2[c2] <= p2) {
+ do {
+ c2++;
+ } while (bucket2[c2] <= p2);
+ }
+ p2 = P[p2];
+ U2[i] = bswap16(c2);
+ u16 c3 = fastbits[p3 >> shift];
+ if (bucket2[c3] <= p3) {
+ do {
+ c3++;
+ } while (bucket2[c3] <= p3);
+ }
+ p3 = P[p3];
+ U3[i] = bswap16(c3);
+ u16 c4 = fastbits[p4 >> shift];
+ if (bucket2[c4] <= p4) {
+ do {
+ c4++;
+ } while (bucket2[c4] <= p4);
+ }
+ p4 = P[p4];
+ U4[i] = bswap16(c4);
+ u16 c5 = fastbits[p5 >> shift];
+ if (bucket2[c5] <= p5) {
+ do {
+ c5++;
+ } while (bucket2[c5] <= p5);
+ }
+ p5 = P[p5];
+ U5[i] = bswap16(c5);
+ u16 c6 = fastbits[p6 >> shift];
+ if (bucket2[c6] <= p6) {
+ do {
+ c6++;
+ } while (bucket2[c6] <= p6);
+ }
+ p6 = P[p6];
+ U6[i] = bswap16(c6);
+ }
+
+ *i0 = p0;
+ *i1 = p1;
+ *i2 = p2;
+ *i3 = p3;
+ *i4 = p4;
+ *i5 = p5;
+ *i6 = p6;
+}
+
+static void libsais_unbwt_decode_8(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, fast_uint_t shift, fast_uint_t r, fast_uint_t * i0,
+ fast_uint_t * i1, fast_uint_t * i2, fast_uint_t * i3, fast_uint_t * i4,
+ fast_uint_t * i5, fast_uint_t * i6, fast_uint_t * i7, fast_uint_t k) {
+ u16 * RESTRICT U0 = (u16 *)(void *)U;
+ u16 * RESTRICT U1 = (u16 *)(void *)(((u8 *)U0) + r);
+ u16 * RESTRICT U2 = (u16 *)(void *)(((u8 *)U1) + r);
+ u16 * RESTRICT U3 = (u16 *)(void *)(((u8 *)U2) + r);
+ u16 * RESTRICT U4 = (u16 *)(void *)(((u8 *)U3) + r);
+ u16 * RESTRICT U5 = (u16 *)(void *)(((u8 *)U4) + r);
+ u16 * RESTRICT U6 = (u16 *)(void *)(((u8 *)U5) + r);
+ u16 * RESTRICT U7 = (u16 *)(void *)(((u8 *)U6) + r);
+
+ fast_uint_t i, p0 = *i0, p1 = *i1, p2 = *i2, p3 = *i3, p4 = *i4, p5 = *i5, p6 = *i6, p7 = *i7;
+
+ for (i = 0; i != k; ++i) {
+ u16 c0 = fastbits[p0 >> shift];
+ if (bucket2[c0] <= p0) {
+ do {
+ c0++;
+ } while (bucket2[c0] <= p0);
+ }
+ p0 = P[p0];
+ U0[i] = bswap16(c0);
+ u16 c1 = fastbits[p1 >> shift];
+ if (bucket2[c1] <= p1) {
+ do {
+ c1++;
+ } while (bucket2[c1] <= p1);
+ }
+ p1 = P[p1];
+ U1[i] = bswap16(c1);
+ u16 c2 = fastbits[p2 >> shift];
+ if (bucket2[c2] <= p2) {
+ do {
+ c2++;
+ } while (bucket2[c2] <= p2);
+ }
+ p2 = P[p2];
+ U2[i] = bswap16(c2);
+ u16 c3 = fastbits[p3 >> shift];
+ if (bucket2[c3] <= p3) {
+ do {
+ c3++;
+ } while (bucket2[c3] <= p3);
+ }
+ p3 = P[p3];
+ U3[i] = bswap16(c3);
+ u16 c4 = fastbits[p4 >> shift];
+ if (bucket2[c4] <= p4) {
+ do {
+ c4++;
+ } while (bucket2[c4] <= p4);
+ }
+ p4 = P[p4];
+ U4[i] = bswap16(c4);
+ u16 c5 = fastbits[p5 >> shift];
+ if (bucket2[c5] <= p5) {
+ do {
+ c5++;
+ } while (bucket2[c5] <= p5);
+ }
+ p5 = P[p5];
+ U5[i] = bswap16(c5);
+ u16 c6 = fastbits[p6 >> shift];
+ if (bucket2[c6] <= p6) {
+ do {
+ c6++;
+ } while (bucket2[c6] <= p6);
+ }
+ p6 = P[p6];
+ U6[i] = bswap16(c6);
+ u16 c7 = fastbits[p7 >> shift];
+ if (bucket2[c7] <= p7) {
+ do {
+ c7++;
+ } while (bucket2[c7] <= p7);
+ }
+ p7 = P[p7];
+ U7[i] = bswap16(c7);
+ }
+
+ *i0 = p0;
+ *i1 = p1;
+ *i2 = p2;
+ *i3 = p3;
+ *i4 = p4;
+ *i5 = p5;
+ *i6 = p6;
+ *i7 = p7;
+}
+
+static void libsais_unbwt_decode(u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_sint_t n, sa_sint_t r,
+ const sa_uint_t * RESTRICT I, sa_uint_t * RESTRICT bucket2, u16 * RESTRICT fastbits,
+ fast_sint_t blocks, fast_uint_t reminder) {
+ fast_uint_t shift = 0;
+ while ((n >> shift) > (1 << UNBWT_FASTBITS)) {
+ shift++;
+ }
+ fast_uint_t offset = 0;
+
+ while (blocks > 8) {
+ fast_uint_t i0 = I[0], i1 = I[1], i2 = I[2], i3 = I[3], i4 = I[4], i5 = I[5], i6 = I[6], i7 = I[7];
+ libsais_unbwt_decode_8(U + offset, P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1, &i2, &i3, &i4, &i5,
+ &i6, &i7, (fast_uint_t)r >> 1);
+ I += 8;
+ blocks -= 8;
+ offset += 8 * (fast_uint_t)r;
+ }
+
+ if (blocks == 1) {
+ fast_uint_t i0 = I[0];
+ libsais_unbwt_decode_1(U + offset, P, bucket2, fastbits, shift, &i0, reminder >> 1);
+ } else if (blocks == 2) {
+ fast_uint_t i0 = I[0], i1 = I[1];
+ libsais_unbwt_decode_2(U + offset, P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1, reminder >> 1);
+ libsais_unbwt_decode_1(U + offset + 2 * (reminder >> 1), P, bucket2, fastbits, shift, &i0,
+ ((fast_uint_t)r >> 1) - (reminder >> 1));
+ } else if (blocks == 3) {
+ fast_uint_t i0 = I[0], i1 = I[1], i2 = I[2];
+ libsais_unbwt_decode_3(U + offset, P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1, &i2, reminder >> 1);
+ libsais_unbwt_decode_2(U + offset + 2 * (reminder >> 1), P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1,
+ ((fast_uint_t)r >> 1) - (reminder >> 1));
+ } else if (blocks == 4) {
+ fast_uint_t i0 = I[0], i1 = I[1], i2 = I[2], i3 = I[3];
+ libsais_unbwt_decode_4(U + offset, P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1, &i2, &i3,
+ reminder >> 1);
+ libsais_unbwt_decode_3(U + offset + 2 * (reminder >> 1), P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1,
+ &i2, ((fast_uint_t)r >> 1) - (reminder >> 1));
+ } else if (blocks == 5) {
+ fast_uint_t i0 = I[0], i1 = I[1], i2 = I[2], i3 = I[3], i4 = I[4];
+ libsais_unbwt_decode_5(U + offset, P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1, &i2, &i3, &i4,
+ reminder >> 1);
+ libsais_unbwt_decode_4(U + offset + 2 * (reminder >> 1), P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1,
+ &i2, &i3, ((fast_uint_t)r >> 1) - (reminder >> 1));
+ } else if (blocks == 6) {
+ fast_uint_t i0 = I[0], i1 = I[1], i2 = I[2], i3 = I[3], i4 = I[4], i5 = I[5];
+ libsais_unbwt_decode_6(U + offset, P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1, &i2, &i3, &i4, &i5,
+ reminder >> 1);
+ libsais_unbwt_decode_5(U + offset + 2 * (reminder >> 1), P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1,
+ &i2, &i3, &i4, ((fast_uint_t)r >> 1) - (reminder >> 1));
+ } else if (blocks == 7) {
+ fast_uint_t i0 = I[0], i1 = I[1], i2 = I[2], i3 = I[3], i4 = I[4], i5 = I[5], i6 = I[6];
+ libsais_unbwt_decode_7(U + offset, P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1, &i2, &i3, &i4, &i5,
+ &i6, reminder >> 1);
+ libsais_unbwt_decode_6(U + offset + 2 * (reminder >> 1), P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1,
+ &i2, &i3, &i4, &i5, ((fast_uint_t)r >> 1) - (reminder >> 1));
+ } else {
+ fast_uint_t i0 = I[0], i1 = I[1], i2 = I[2], i3 = I[3], i4 = I[4], i5 = I[5], i6 = I[6], i7 = I[7];
+ libsais_unbwt_decode_8(U + offset, P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1, &i2, &i3, &i4, &i5,
+ &i6, &i7, reminder >> 1);
+ libsais_unbwt_decode_7(U + offset + 2 * (reminder >> 1), P, bucket2, fastbits, shift, (fast_uint_t)r, &i0, &i1,
+ &i2, &i3, &i4, &i5, &i6, ((fast_uint_t)r >> 1) - (reminder >> 1));
+ }
+}
+
+static void libsais_unbwt_decode_omp(const u8 * RESTRICT T, u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_sint_t n,
+ sa_sint_t r, const sa_uint_t * RESTRICT I, sa_uint_t * RESTRICT bucket2,
+ u16 * RESTRICT fastbits, sa_sint_t threads) {
+ fast_uint_t lastc = T[0];
+ fast_sint_t blocks = 1 + (((fast_sint_t)n - 1) / (fast_sint_t)r);
+ fast_uint_t reminder = (fast_uint_t)n - ((fast_uint_t)r * ((fast_uint_t)blocks - 1));
+
+ {
+ (void)(threads);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+ fast_sint_t omp_block_stride = blocks / omp_num_threads;
+ fast_sint_t omp_block_reminder = blocks % omp_num_threads;
+ fast_sint_t omp_block_size = omp_block_stride + (omp_thread_num < omp_block_reminder);
+ fast_sint_t omp_block_start = omp_block_stride * omp_thread_num +
+ (omp_thread_num < omp_block_reminder ? omp_thread_num : omp_block_reminder);
+
+ libsais_unbwt_decode(U + r * omp_block_start, P, n, r, I + omp_block_start, bucket2, fastbits, omp_block_size,
+ omp_thread_num < omp_num_threads - 1 ? (fast_uint_t)r : reminder);
+ }
+
+ U[n - 1] = (u8)lastc;
+}
+
+static sa_sint_t libsais_unbwt_core(const u8 * RESTRICT T, u8 * RESTRICT U, sa_uint_t * RESTRICT P, sa_sint_t n,
+ const sa_sint_t * freq, sa_sint_t r, const sa_uint_t * RESTRICT I,
+ sa_uint_t * RESTRICT bucket2, u16 * RESTRICT fastbits, sa_uint_t * RESTRICT buckets,
+ sa_sint_t threads) {
+ (void)(buckets);
+
+ { libsais_unbwt_init_single(T, P, n, freq, I, bucket2, fastbits); }
+
+ libsais_unbwt_decode_omp(T, U, P, n, r, I, bucket2, fastbits, threads);
+ return 0;
+}
+
+static sa_sint_t libsais_unbwt_main(const u8 * T, u8 * U, sa_uint_t * P, sa_sint_t n, const sa_sint_t * freq,
+ sa_sint_t r, const sa_uint_t * I, sa_sint_t threads) {
+ fast_uint_t shift = 0;
+ while ((n >> shift) > (1 << UNBWT_FASTBITS)) {
+ shift++;
+ }
+
+ sa_uint_t * RESTRICT bucket2 =
+ (sa_uint_t *)libsais_alloc_aligned(ALPHABET_SIZE * ALPHABET_SIZE * sizeof(sa_uint_t), 4096);
+ u16 * RESTRICT fastbits = (u16 *)libsais_alloc_aligned(((size_t)1 + (size_t)(n >> shift)) * sizeof(u16), 4096);
+ memset(fastbits, 0, ((size_t)1 + (size_t)(n >> shift)) * sizeof(u16));
+ sa_uint_t * RESTRICT buckets =
+ threads > 1 && n >= 262144
+ ? (sa_uint_t *)libsais_alloc_aligned(
+ (size_t)threads * (ALPHABET_SIZE + (ALPHABET_SIZE * ALPHABET_SIZE)) * sizeof(sa_uint_t), 4096)
+ : NULL;
+
+ sa_sint_t index = bucket2 != NULL && fastbits != NULL && (buckets != NULL || threads == 1 || n < 262144)
+ ? libsais_unbwt_core(T, U, P, n, freq, r, I, bucket2, fastbits, buckets, threads)
+ : -2;
+
+ libsais_free_aligned(buckets);
+ libsais_free_aligned(fastbits);
+ libsais_free_aligned(bucket2);
+
+ return index;
+}
+
+static sa_sint_t libsais_unbwt_main_ctx(const LIBSAIS_UNBWT_CONTEXT * ctx, const u8 * T, u8 * U, sa_uint_t * P,
+ sa_sint_t n, const sa_sint_t * freq, sa_sint_t r, const sa_uint_t * I) {
+ return ctx != NULL && ctx->bucket2 != NULL && ctx->fastbits != NULL && (ctx->buckets != NULL || ctx->threads == 1)
+ ? libsais_unbwt_core(T, U, P, n, freq, r, I, ctx->bucket2, ctx->fastbits, ctx->buckets,
+ (sa_sint_t)ctx->threads)
+ : -2;
+}
+
+static void * libsais_unbwt_create_ctx(void) { return (void *)libsais_unbwt_create_ctx_main(1); }
+
+static void libsais_unbwt_free_ctx(void * ctx) { libsais_unbwt_free_ctx_main((LIBSAIS_UNBWT_CONTEXT *)ctx); }
+
+static s32 libsais_unbwt_aux(const u8 * T, u8 * U, s32 * A, s32 n, const s32 * freq, s32 r, const s32 * I) {
+ if ((T == NULL) || (U == NULL) || (A == NULL) || (n < 0) || ((r != n) && ((r < 2) || ((r & (r - 1)) != 0))) ||
+ (I == NULL)) {
+ return -1;
+ } else if (n <= 1) {
+ if (I[0] != n) {
+ return -1;
+ }
+ if (n == 1) {
+ U[0] = T[0];
+ }
+ return 0;
+ }
+
+ fast_sint_t t;
+ for (t = 0; t <= (n - 1) / r; ++t) {
+ if (I[t] <= 0 || I[t] > n) {
+ return -1;
+ }
+ }
+
+ return libsais_unbwt_main(T, U, (sa_uint_t *)A, n, freq, r, (const sa_uint_t *)I, 1);
+}
+
+static s32 libsais_unbwt_aux_ctx(const void * ctx, const u8 * T, u8 * U, s32 * A, s32 n, const s32 * freq, s32 r,
+ const s32 * I) {
+ if ((T == NULL) || (U == NULL) || (A == NULL) || (n < 0) || ((r != n) && ((r < 2) || ((r & (r - 1)) != 0))) ||
+ (I == NULL)) {
+ return -1;
+ } else if (n <= 1) {
+ if (I[0] != n) {
+ return -1;
+ }
+ if (n == 1) {
+ U[0] = T[0];
+ }
+ return 0;
+ }
+
+ fast_sint_t t;
+ for (t = 0; t <= (n - 1) / r; ++t) {
+ if (I[t] <= 0 || I[t] > n) {
+ return -1;
+ }
+ }
+
+ return libsais_unbwt_main_ctx((const LIBSAIS_UNBWT_CONTEXT *)ctx, T, U, (sa_uint_t *)A, n, freq, r,
+ (const sa_uint_t *)I);
+}
+
+static s32 libsais_unbwt(const u8 * T, u8 * U, s32 * A, s32 n, const s32 * freq, s32 i) {
+ return libsais_unbwt_aux(T, U, A, n, freq, n, &i);
+}
+
+static s32 libsais_unbwt_ctx(const void * ctx, const u8 * T, u8 * U, s32 * A, s32 n, const s32 * freq, s32 i) {
+ return libsais_unbwt_aux_ctx(ctx, T, U, A, n, freq, n, &i);
+}
+
+static void libsais_compute_phi(const sa_sint_t * RESTRICT SA, sa_sint_t * RESTRICT PLCP, sa_sint_t n,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ sa_sint_t k = omp_block_start > 0 ? SA[omp_block_start - 1] : n;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 3; i < j; i += 4) {
+ prefetchw(&PLCP[SA[i + prefetch_distance + 0]]);
+ prefetchw(&PLCP[SA[i + prefetch_distance + 1]]);
+
+ PLCP[SA[i + 0]] = k;
+ k = SA[i + 0];
+ PLCP[SA[i + 1]] = k;
+ k = SA[i + 1];
+
+ prefetchw(&PLCP[SA[i + prefetch_distance + 2]]);
+ prefetchw(&PLCP[SA[i + prefetch_distance + 3]]);
+
+ PLCP[SA[i + 2]] = k;
+ k = SA[i + 2];
+ PLCP[SA[i + 3]] = k;
+ k = SA[i + 3];
+ }
+
+ for (j += prefetch_distance + 3; i < j; i += 1) {
+ PLCP[SA[i]] = k;
+ k = SA[i];
+ }
+}
+
+static void libsais_compute_phi_omp(const sa_sint_t * RESTRICT SA, sa_sint_t * RESTRICT PLCP, sa_sint_t n,
+ sa_sint_t threads) {
+ {
+ (void)(threads);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (n / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : n - omp_block_start;
+
+ libsais_compute_phi(SA, PLCP, n, omp_block_start, omp_block_size);
+ }
+}
+
+static void libsais_compute_plcp(const u8 * RESTRICT T, sa_sint_t * RESTRICT PLCP, fast_sint_t n,
+ fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j, l = 0;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance; i < j; i += 1) {
+ prefetch(&T[PLCP[i + prefetch_distance] + l]);
+
+ fast_sint_t k = PLCP[i], m = n - (i > k ? i : k);
+ while (l < m && T[i + l] == T[k + l]) {
+ l++;
+ }
+
+ PLCP[i] = (sa_sint_t)l;
+ l -= (l != 0);
+ }
+
+ for (j += prefetch_distance; i < j; i += 1) {
+ fast_sint_t k = PLCP[i], m = n - (i > k ? i : k);
+ while (l < m && T[i + l] == T[k + l]) {
+ l++;
+ }
+
+ PLCP[i] = (sa_sint_t)l;
+ l -= (l != 0);
+ }
+}
+
+static void libsais_compute_plcp_omp(const u8 * RESTRICT T, sa_sint_t * RESTRICT PLCP, sa_sint_t n, sa_sint_t threads) {
+ {
+ (void)(threads);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (n / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : n - omp_block_start;
+
+ libsais_compute_plcp(T, PLCP, n, omp_block_start, omp_block_size);
+ }
+}
+
+static void libsais_compute_lcp(const sa_sint_t * RESTRICT PLCP, const sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT LCP, fast_sint_t omp_block_start, fast_sint_t omp_block_size) {
+ const fast_sint_t prefetch_distance = 32;
+
+ fast_sint_t i, j;
+ for (i = omp_block_start, j = omp_block_start + omp_block_size - prefetch_distance - 3; i < j; i += 4) {
+ prefetch(&PLCP[SA[i + prefetch_distance + 0]]);
+ prefetch(&PLCP[SA[i + prefetch_distance + 1]]);
+
+ LCP[i + 0] = PLCP[SA[i + 0]];
+ LCP[i + 1] = PLCP[SA[i + 1]];
+
+ prefetch(&PLCP[SA[i + prefetch_distance + 2]]);
+ prefetch(&PLCP[SA[i + prefetch_distance + 3]]);
+
+ LCP[i + 2] = PLCP[SA[i + 2]];
+ LCP[i + 3] = PLCP[SA[i + 3]];
+ }
+
+ for (j += prefetch_distance + 3; i < j; i += 1) {
+ LCP[i] = PLCP[SA[i]];
+ }
+}
+
+static void libsais_compute_lcp_omp(const sa_sint_t * RESTRICT PLCP, const sa_sint_t * RESTRICT SA,
+ sa_sint_t * RESTRICT LCP, sa_sint_t n, sa_sint_t threads) {
+ {
+ (void)(threads);
+
+ fast_sint_t omp_thread_num = 0;
+ fast_sint_t omp_num_threads = 1;
+
+ fast_sint_t omp_block_stride = (n / omp_num_threads) & (-16);
+ fast_sint_t omp_block_start = omp_thread_num * omp_block_stride;
+ fast_sint_t omp_block_size = omp_thread_num < omp_num_threads - 1 ? omp_block_stride : n - omp_block_start;
+
+ libsais_compute_lcp(PLCP, SA, LCP, omp_block_start, omp_block_size);
+ }
+}
+
+static s32 libsais_plcp(const u8 * T, const s32 * SA, s32 * PLCP, s32 n) {
+ if ((T == NULL) || (SA == NULL) || (PLCP == NULL) || (n < 0)) {
+ return -1;
+ } else if (n <= 1) {
+ if (n == 1) {
+ PLCP[0] = 0;
+ }
+ return 0;
+ }
+
+ libsais_compute_phi_omp(SA, PLCP, n, 1);
+ libsais_compute_plcp_omp(T, PLCP, n, 1);
+
+ return 0;
+}
+
+static s32 libsais_lcp(const s32 * PLCP, const s32 * SA, s32 * LCP, s32 n) {
+ if ((PLCP == NULL) || (SA == NULL) || (LCP == NULL) || (n < 0)) {
+ return -1;
+ } else if (n <= 1) {
+ if (n == 1) {
+ LCP[0] = PLCP[SA[0]];
+ }
+ return 0;
+ }
+
+ libsais_compute_lcp_omp(PLCP, SA, LCP, n, 1);
+
+ return 0;
+}
+
+#endif
diff --git a/src/Codec/Bz3.hs b/src/Codec/Bz3.hs
new file mode 100644
--- /dev/null
+++ b/src/Codec/Bz3.hs
@@ -0,0 +1,90 @@
+module Codec.Bz3 ( Bz3Error (..)
+ , decompressFile
+ , compressFile
+ ) where
+
+import Codec.Bz3.Binary
+import Codec.Bz3.Foreign
+import qualified Control.Monad.ST.Lazy as LazyST
+import qualified Control.Monad.ST.Lazy.Unsafe as LazyST
+import Data.Bifunctor (bimap)
+import Data.Binary.Get (runGetOrFail)
+import Data.Binary.Put (runPut)
+import qualified Data.ByteString as BS
+import Data.ByteString.Internal as BS
+import qualified Data.ByteString.Lazy as BSL
+import qualified Data.ByteString.Unsafe as BS
+import Data.Int (Int32)
+import Foreign.ForeignPtr (ForeignPtr, castForeignPtr,
+ mallocForeignPtrBytes,
+ newForeignPtr, withForeignPtr)
+import Foreign.Marshal.Array (copyArray)
+import Foreign.Ptr (castPtr)
+
+newBz3StForeign :: Int32 -> IO (ForeignPtr Bz3St)
+newBz3StForeign oSz = castForeignPtr <$> (newForeignPtr bz3Free . castPtr =<< bz3New oSz)
+
+decompressFile :: BSL.ByteString -> BSL.ByteString
+decompressFile contents = LazyST.runST $ BSL.fromChunks <$> do
+ let Right (bs, _, bSz) = runGetOrFail getFileH contents
+ st <- LazyST.unsafeIOToST $ newBz3StForeign (fromIntegral bSz)
+ loop st bs
+ where
+ loop st bs | BSL.null bs = pure []
+ | otherwise = do {(dc, rest) <- decNext st bs; (dc:) <$> loop st rest}
+
+-- will fail if bz3st cannot fit exotic big input
+compressFile :: BSL.ByteString -> BSL.ByteString
+compressFile d = LazyST.runST $ do
+ st <- LazyST.unsafeIOToST $ newBz3StForeign bSz
+ (sz,bb) <- loop st (BSL.toChunks d)
+ let fileH = runPut (putFileH sz)
+ pure (fileH<>BSL.fromChunks bb)
+ where
+ loop _ [] = pure (0, [])
+ loop s (b:bs) | BS.length b > feed = let (next,b') = BS.splitAt feed b in loop s (next:b':bs)
+ loop s (b:bs) = do
+ (csz, e) <- if osz<64
+ then pure (0xffffffff, b)
+ else encN s b
+ let chunk=BSL.toStrict $ runPut (putChunk (Chunk (fromIntegral csz) osz))
+ bimap (max osz) ((chunk:).(e:)) <$> loop s bs
+ where
+ osz=fromIntegral (BS.length b)
+
+ feed=fromIntegral (bz3Bound bSz)
+
+ bSz :: Integral a => a
+ bSz=16*1024*1024
+
+encN :: ForeignPtr Bz3St -> BS.ByteString -> LazyST.ST s (Int32, BS.ByteString)
+encN st inp = LazyST.unsafeIOToST $
+ BS.unsafeUseAsCStringLen inp $ \(d,sz) -> do
+ let bufSz = bz3Bound (fromIntegral sz)
+ buf <- mallocForeignPtrBytes (fromIntegral bufSz)
+ enc <- withForeignPtr buf $ \bb -> do
+ copyArray bb d sz
+ bz3EncodeBlock st (castPtr bb) (fromIntegral sz)
+ pure (enc, BS.BS (castForeignPtr buf) (fromIntegral enc))
+
+decNext :: ForeignPtr Bz3St -> BSL.ByteString -> LazyST.ST s (BS.ByteString, BSL.ByteString)
+decNext st inp = LazyST.unsafeIOToST $ do
+ let (next, off, Chunk csz osz) = y (runGetOrFail getChunk inp)
+ if osz>=64
+ then do
+ let bufSz = bz3Bound (fromIntegral osz)
+ csz32 = fromIntegral csz; osz32 = fromIntegral osz
+ buf <- mallocForeignPtrBytes (fromIntegral bufSz)
+ let bb=BSL.toStrict (BSL.drop 8 $ BSL.take off inp)
+ res <- withForeignPtr buf $ \b -> do
+ BS.unsafeUseAsCStringLen bb $ \(p,isz) -> copyArray b (castPtr p) isz
+ bz3DecodeBlock st b bufSz csz32 osz32
+ case res of
+ Right{} -> pure ()
+ Left e | e==Bz3Ok -> pure ()
+ | otherwise -> error =<< bz3Strerror st
+ pure (BS.BS (castForeignPtr buf) (fromIntegral osz), next)
+ else let bb=BSL.toStrict (BSL.drop 8 $ BSL.take off inp)
+ in pure (bb, next)
+ where
+ y (Right x) = x; y (Left (_, _, e)) = error e
diff --git a/src/Codec/Bz3/Binary.hs b/src/Codec/Bz3/Binary.hs
new file mode 100644
--- /dev/null
+++ b/src/Codec/Bz3/Binary.hs
@@ -0,0 +1,55 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+-- https://github.com/kspalaiologos/bzip3/blob/master/doc/bzip3_format.md
+module Codec.Bz3.Binary ( Chunk (..)
+ , getFileH
+ , getFrameH
+ , getChunk
+ , putFileH
+ , putChunk
+ ) where
+
+import Control.Monad (unless, when)
+import Data.Binary.Get (Get, getByteString, getWord32le, getWord8,
+ skip)
+import Data.Binary.Put (Put, putByteString, putWord32le)
+import Data.Bits (popCount, (.&.))
+import Data.Word (Word32)
+
+data Chunk = Chunk { compressedSz, origSz :: !Word32 }
+
+putFileH :: Word32 -> Put
+putFileH maxSz = do
+ putByteString "BZ3v1"
+ putWord32le maxSz
+
+-- max block sz
+getFileH, getFrameH :: Get Word32
+getFileH = do {sig <- getByteString 5; unless (sig=="BZ3v1") $ fail "bad signature"; getWord32le}
+
+getFrameH = getFileH <* getWord32le
+
+putChunk :: Chunk -> Put
+putChunk (Chunk csz osz) = putWord32le csz *> putWord32le osz
+
+getChunk :: Get Chunk
+getChunk = do
+ csz <- getWord32le; osz <- getWord32le
+ if osz<64
+ then getSmallBlock csz
+ else getRegularBlock csz
+ pure $ Chunk csz osz
+
+getSmallBlock :: Word32 -> Get ()
+getSmallBlock sz = do
+ skip 4; lit <- getWord32le
+ unless (lit==0xffffffff) $ fail "Small block not expected to have bwtIx"
+ skip (fromIntegral sz-8)
+
+getRegularBlock :: Word32 -> Get ()
+getRegularBlock sz = do
+ skip 8
+ model <- getWord8
+ when (model .&. 0x2 /= 0) (skip 4)
+ when (model .&. 0x4 /= 0) (skip 4)
+ skip (fromIntegral sz - (popCount model*4 + 9))
diff --git a/src/Codec/Bz3/Foreign.chs b/src/Codec/Bz3/Foreign.chs
new file mode 100644
--- /dev/null
+++ b/src/Codec/Bz3/Foreign.chs
@@ -0,0 +1,49 @@
+module Codec.Bz3.Foreign ( Bz3St
+ , Bz3Error (..)
+ , bz3New
+ , bz3Free
+ , bz3Bound
+ , bz3Strerror
+ , bz3DecodeBlock
+ , bz3EncodeBlock
+ ) where
+
+import Data.Int (Int32)
+import Foreign.Ptr (Ptr)
+import Foreign.C.Types (CSize)
+
+#include
+
+{# enum define Bz3Error { BZ3_OK as Bz3Ok
+ , BZ3_ERR_OUT_OF_BOUNDS as Bz3ErrOutOfBounds
+ , BZ3_ERR_BWT as Bz3ErrBwt
+ , BZ3_ERR_CRC as Bz3ErrCrc
+ , BZ3_ERR_MALFORMED_HEADER as Bz3ErrMalformedHeader
+ , BZ3_ERR_TRUNCATED_DATA as Bz3ErrTruncatedData
+ , BZ3_ERR_DATA_TOO_BIG as Bz3ErrDataTooBig
+ , BZ3_ERR_INIT as Bz3ErrInit
+ } deriving (Eq)
+ #}
+
+data Bz3St
+
+{# pointer *bz3_state as Bz3StPtr foreign finalizer bz3_free as ^ -> Bz3St #}
+
+type UInt8 = {# type uint8_t #}
+{#typedef uint8_t UInt8#}
+{#default in `Ptr UInt8' [uint8_t*] id#}
+
+{#typedef int32_t Int32#}
+
+{#typedef size_t CSize#}
+{#default in `Ptr CSize' [size_t*] id#}
+
+{# fun bz3_new as ^ { `Int32' } -> `Ptr Bz3St' id #}
+{# fun pure bz3_bound as ^ { `CSize' } -> `CSize' #}
+{# fun bz3_strerror as ^ { `Bz3StPtr' } -> `String' #}
+{# fun bz3_decode_block as ^ { `Bz3StPtr', `Ptr UInt8', `CSize', `Int32', `Int32' } -> `Either Bz3Error Int32' bz3Err #}
+{# fun bz3_encode_block as ^ { `Bz3StPtr', `Ptr UInt8', `Int32' } -> `Int32' #}
+
+bz3Err :: Int32 -> Either Bz3Error Int32
+bz3Err i | i > 0 = Right i
+ | otherwise = Left$toEnum (fromIntegral i)
diff --git a/test/Main.hs b/test/Main.hs
new file mode 100644
--- /dev/null
+++ b/test/Main.hs
@@ -0,0 +1,19 @@
+module Main (main) where
+
+import Codec.Bz3
+import qualified Data.ByteString.Lazy as BSL
+import System.Directory (doesDirectoryExist, listDirectory)
+import Test.Tasty (defaultMain, testGroup)
+import Test.Tasty.HUnit (assertBool, testCase)
+
+main :: IO ()
+main = do
+ d <- do {e <- doesDirectoryExist "test/data"; if e then listDirectory "test/data" else pure []}
+ defaultMain $ testGroup "bz3" [ roundtrip ("test/data/" ++ f) | f <- d ]
+
+roundtrip fp = testCase ("roundtrip " ++ fp) $ do
+ contents <- BSL.readFile fp
+ let d = decompressFile contents
+ e = compressFile d
+ d1 = decompressFile e
+ assertBool "doesn't error" (d1 `seq` True)