diff --git a/Changelog.md b/Changelog.md --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,112 @@ # Changelog for `proto-lens` -## Unreleased changes +## v0.7.1.7 +- Relax upper bounds for ghc-9.10. +- Bump upper bound to allow ghc-prim-0.12 (ghc-9.10.2). +- Support GHC 9.12. +- Support GHC 9.14. + +## v0.7.1.2 +- Support GHC 9.4 + +## v0.7.1.1 +- Relax upper bounds for ghc-9.2 + +## v0.7.1.0 +- Support GHC 9.0. +- Fixed parsing of UTF8 chars in text format proto string literals. +- Support all valid text format boolean field values +- Remove redundant (.&. 127) from putVarInt (#396) + +## v0.7.0.0 +- Support GHC 8.10. +- Add a method to `Data.ProtoLens.Message` for getting the `DescriptorProto` + of a given message. For a simpler API, see `Data.ProtoLens.Descriptor` + from `proto-lens-protobuf-types`. +- Bump ghc-source-gen to version 0.4.0.0. +- Bump upper bound to allow base-4.14. +- Bump upper bound to allow ghc-prim-0.6. + +## v0.6.0.0 + +### Breaking Changes +- Bump lower bounds to base-4.10 (ghc-8.2). +- Move `Proto.Google.Protobuf.Descriptor` and + `Proto.Google.Protobuf.Compiler.Plugin` to the `proto-lens-protobuf-types` + package. + +### Backwards-Compatible Changes +- Bump upper bound to allow profunctors-5.5. +- Support dependencies on base-4.13 (ghc-8.8) and lens-family-2.0. + +## v0.5.1.0 +- Add `decodeMessageDelimitedH` to decode delimited messages from a file handle + (#61). + +## v0.5.0.1 +- Bump the upper bound on `profunctors` to allow 5.4. +- Bump the upper bound on `primitive` to allow 0.7. +- Allow text format protobuf strings to contain unescaped quote characters + different from the delimiters (#320). + +### Breaking Changes +- Merge the `lens-labels` library into `proto-lens`: + - `Lens.Labels` => `Data.ProtoLens.Fields` + - `Lens.Labels.Unwrapped` => `Data.ProtoLens.Labels` + - `Lens.Labels.Prism` => `Data.ProtoLens.Prism` +- Simplify the API of `Data.ProtoLens.Encoding.Wire`, using a plain ADT + rather than a GADT to represent unknown field values. +- If fields have the wrong wire type, store them in `unknownFields` rather + than failing the parse. (#125) + +### Backwards-Compatible Changes +- Merge proto-lens-combinators into the proto-lens library. +- Use generated Haskell code to encode/decode proto messages more quickly. + In particular: + - Add the methods `parseMessage` and `buildMessage` to `Data.ProtoLens.Message` + - Expose an opaque `Parser` monad, which is used by the generated + code. + - Expose the module `Data.ProtoLens.Encoding.Bytes`, which is used + by the generated code. + - Various other, related performance optimizations. +- Add functionality for storing unknown fields as `Vector`s. (See the + changelog of `proto-lens-protoc` for more details.) Exposes the + `Growing` type for mutable vectors of growing capacity. +- Export the new function `parseMessageDelimited`. (#61) + +## v0.4.0.1 +- Bump the dependencies on `base` and `containers` to `0.4.0.1`. + +## v0.4.0.0 +- Don't use `data-default` for default proto values (#194). +- Update the proto descriptors to protobuf-3.6.1. +- Use simplified lens-labels instances. (#208) + +## v0.3.1.1 +- Bump the lower bound on `base` to indicate we require `ghc>=8.0`. + +## v0.3.1.0 +- Improve references to types/fields in decoding error messages (#187). +- Bump the dependency on `base` for `ghc-8.4.2`. +- Make `Registry` an instance of `Semigroup`. + +## v0.3.0.0 +- Remove support for `ghc-7.10`. (#136) +- Use a `.cabal` file that's auto-generated from `hpack`. (#138) +- Add buildMessageDelimited: size-delimited streams of Messages (#102) +- Add support for parsing `Any` messages in google protobuf text format (#124) +- Use the Tag newtype consistently. (#127) +- Add support for tracking unknown fields. (#129) +- Improve an error message. (#132) +- Bundle enum pattern synonyms with their type. (#136) +- Implement proto3-style "open" enums. (#137) +- Consolidate `proto-lens-descriptors` into `proto-lens`. (#140) +- Split the `Message` class into separate methods. (#139) +- Improve an error message when decoding Anys. (#146) +- Refactor the `FieldDescriptorType. (#147) +- Improve text format error messages. (#148) +- Add module `Data.ProtoLens.Service.Types`. (#154) +- Add Haddock comments to fields. (#172) ## v0.2.2.0 - Bump the dependency on `base` to support `ghc-8.2.1`. diff --git a/proto-lens-imports/google/protobuf/compiler/plugin.proto b/proto-lens-imports/google/protobuf/compiler/plugin.proto new file mode 100644 --- /dev/null +++ b/proto-lens-imports/google/protobuf/compiler/plugin.proto @@ -0,0 +1,180 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Author: kenton@google.com (Kenton Varda) +// +// protoc (aka the Protocol Compiler) can be extended via plugins. A plugin is +// just a program that reads a CodeGeneratorRequest from stdin and writes a +// CodeGeneratorResponse to stdout. +// +// Plugins written using C++ can use google/protobuf/compiler/plugin.h instead +// of dealing with the raw protocol defined here. +// +// A plugin executable needs only to be placed somewhere in the path. The +// plugin should be named "protoc-gen-$NAME", and will then be used when the +// flag "--${NAME}_out" is passed to protoc. + +syntax = "proto2"; + +package google.protobuf.compiler; +option java_package = "com.google.protobuf.compiler"; +option java_outer_classname = "PluginProtos"; + +option csharp_namespace = "Google.Protobuf.Compiler"; +option go_package = "google.golang.org/protobuf/types/pluginpb"; + +import "google/protobuf/descriptor.proto"; + +// The version number of protocol compiler. +message Version { + optional int32 major = 1; + optional int32 minor = 2; + optional int32 patch = 3; + // A suffix for alpha, beta or rc release, e.g., "alpha-1", "rc2". It should + // be empty for mainline stable releases. + optional string suffix = 4; +} + +// An encoded CodeGeneratorRequest is written to the plugin's stdin. +message CodeGeneratorRequest { + // The .proto files that were explicitly listed on the command-line. The + // code generator should generate code only for these files. Each file's + // descriptor will be included in proto_file, below. + repeated string file_to_generate = 1; + + // The generator parameter passed on the command-line. + optional string parameter = 2; + + // FileDescriptorProtos for all files in files_to_generate and everything + // they import. The files will appear in topological order, so each file + // appears before any file that imports it. + // + // Note: the files listed in files_to_generate will include runtime-retention + // options only, but all other files will include source-retention options. + // The source_file_descriptors field below is available in case you need + // source-retention options for files_to_generate. + // + // protoc guarantees that all proto_files will be written after + // the fields above, even though this is not technically guaranteed by the + // protobuf wire format. This theoretically could allow a plugin to stream + // in the FileDescriptorProtos and handle them one by one rather than read + // the entire set into memory at once. However, as of this writing, this + // is not similarly optimized on protoc's end -- it will store all fields in + // memory at once before sending them to the plugin. + // + // Type names of fields and extensions in the FileDescriptorProto are always + // fully qualified. + repeated FileDescriptorProto proto_file = 15; + + // File descriptors with all options, including source-retention options. + // These descriptors are only provided for the files listed in + // files_to_generate. + repeated FileDescriptorProto source_file_descriptors = 17; + + // The version number of protocol compiler. + optional Version compiler_version = 3; +} + +// The plugin writes an encoded CodeGeneratorResponse to stdout. +message CodeGeneratorResponse { + // Error message. If non-empty, code generation failed. The plugin process + // should exit with status code zero even if it reports an error in this way. + // + // This should be used to indicate errors in .proto files which prevent the + // code generator from generating correct code. Errors which indicate a + // problem in protoc itself -- such as the input CodeGeneratorRequest being + // unparseable -- should be reported by writing a message to stderr and + // exiting with a non-zero status code. + optional string error = 1; + + // A bitmask of supported features that the code generator supports. + // This is a bitwise "or" of values from the Feature enum. + optional uint64 supported_features = 2; + + // Sync with code_generator.h. + enum Feature { + FEATURE_NONE = 0; + FEATURE_PROTO3_OPTIONAL = 1; + FEATURE_SUPPORTS_EDITIONS = 2; + } + + // The minimum edition this plugin supports. This will be treated as an + // Edition enum, but we want to allow unknown values. It should be specified + // according the edition enum value, *not* the edition number. Only takes + // effect for plugins that have FEATURE_SUPPORTS_EDITIONS set. + optional int32 minimum_edition = 3; + + // The maximum edition this plugin supports. This will be treated as an + // Edition enum, but we want to allow unknown values. It should be specified + // according the edition enum value, *not* the edition number. Only takes + // effect for plugins that have FEATURE_SUPPORTS_EDITIONS set. + optional int32 maximum_edition = 4; + + // Represents a single generated file. + message File { + // The file name, relative to the output directory. The name must not + // contain "." or ".." components and must be relative, not be absolute (so, + // the file cannot lie outside the output directory). "/" must be used as + // the path separator, not "\". + // + // If the name is omitted, the content will be appended to the previous + // file. This allows the generator to break large files into small chunks, + // and allows the generated text to be streamed back to protoc so that large + // files need not reside completely in memory at one time. Note that as of + // this writing protoc does not optimize for this -- it will read the entire + // CodeGeneratorResponse before writing files to disk. + optional string name = 1; + + // If non-empty, indicates that the named file should already exist, and the + // content here is to be inserted into that file at a defined insertion + // point. This feature allows a code generator to extend the output + // produced by another code generator. The original generator may provide + // insertion points by placing special annotations in the file that look + // like: + // @@protoc_insertion_point(NAME) + // The annotation can have arbitrary text before and after it on the line, + // which allows it to be placed in a comment. NAME should be replaced with + // an identifier naming the point -- this is what other generators will use + // as the insertion_point. Code inserted at this point will be placed + // immediately above the line containing the insertion point (thus multiple + // insertions to the same point will come out in the order they were added). + // The double-@ is intended to make it unlikely that the generated code + // could contain things that look like insertion points by accident. + // + // For example, the C++ code generator places the following line in the + // .pb.h files that it generates: + // // @@protoc_insertion_point(namespace_scope) + // This line appears within the scope of the file's package namespace, but + // outside of any particular class. Another plugin can then specify the + // insertion_point "namespace_scope" to generate additional classes or + // other declarations that should be placed in this scope. + // + // Note that if the line containing the insertion point begins with + // whitespace, the same whitespace will be added to every line of the + // inserted text. This is useful for languages like Python, where + // indentation matters. In these languages, the insertion point comment + // should be indented the same amount as any inserted code will need to be + // in order to work correctly in that context. + // + // The code generator that generates the initial file and the one which + // inserts into it must both run as part of a single invocation of protoc. + // Code generators are executed in the order in which they appear on the + // command line. + // + // If |insertion_point| is present, |name| must also be present. + optional string insertion_point = 2; + + // The file contents. + optional string content = 15; + + // Information describing the file content being inserted. If an insertion + // point is used, this information will be appropriately offset and inserted + // into the code generation metadata for the generated files. + optional GeneratedCodeInfo generated_code_info = 16; + } + repeated File file = 15; +} diff --git a/proto-lens-imports/google/protobuf/descriptor.proto b/proto-lens-imports/google/protobuf/descriptor.proto new file mode 100644 --- /dev/null +++ b/proto-lens-imports/google/protobuf/descriptor.proto @@ -0,0 +1,1296 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + +syntax = "proto2"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// The full set of known editions. +enum Edition { + // A placeholder for an unknown edition value. + EDITION_UNKNOWN = 0; + + // A placeholder edition for specifying default behaviors *before* a feature + // was first introduced. This is effectively an "infinite past". + EDITION_LEGACY = 900; + + // Legacy syntax "editions". These pre-date editions, but behave much like + // distinct editions. These can't be used to specify the edition of proto + // files, but feature definitions must supply proto2/proto3 defaults for + // backwards compatibility. + EDITION_PROTO2 = 998; + EDITION_PROTO3 = 999; + + // Editions that have been released. The specific values are arbitrary and + // should not be depended on, but they will always be time-ordered for easy + // comparison. + EDITION_2023 = 1000; + EDITION_2024 = 1001; + + // Placeholder editions for testing feature resolution. These should not be + // used or relyed on outside of tests. + EDITION_1_TEST_ONLY = 1; + EDITION_2_TEST_ONLY = 2; + EDITION_99997_TEST_ONLY = 99997; + EDITION_99998_TEST_ONLY = 99998; + EDITION_99999_TEST_ONLY = 99999; + + // Placeholder for specifying unbounded edition support. This should only + // ever be used by plugins that can expect to never require any changes to + // support a new edition. + EDITION_MAX = 0x7FFFFFFF; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field without harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2", "proto3", and "editions". + // + // If `edition` is present, this value must be "editions". + optional string syntax = 12; + + // The edition of the proto file. + optional Edition edition = 14; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + + repeated OneofDescriptorProto oneof_decl = 8; + + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + message Declaration { + // The extension number declared within the extension range. + optional int32 number = 1; + + // The fully-qualified name of the extension field. There must be a leading + // dot in front of the full name. + optional string full_name = 2; + + // The fully-qualified type name of the extension field. Unlike + // Metadata.type, Declaration.type must have a leading dot for messages + // and enums. + optional string type = 3; + + // If true, indicates that the number is reserved in the extension range, + // and any extension field with the number will fail to compile. Set this + // when a declared extension field is deleted. + optional bool reserved = 5; + + // If true, indicates that the extension must be defined as repeated. + // Otherwise the extension must be defined as optional. + optional bool repeated = 6; + + reserved 4; // removed is_repeated + } + + // For external users: DO NOT USE. We are in the process of open sourcing + // extension declaration and executing internal cleanups before it can be + // used externally. + repeated Declaration declaration = 2 [retention = RETENTION_SOURCE]; + + // Any features defined in the specific edition. + optional FeatureSet features = 50; + + // The verification state of the extension range. + enum VerificationState { + // All the extensions of the range must be declared. + DECLARATION = 0; + UNVERIFIED = 1; + } + + // The verification state of the range. + // TODO: flip the default to DECLARATION once all empty ranges + // are marked as UNVERIFIED. + optional VerificationState verification = 3 + [default = UNVERIFIED, retention = RETENTION_SOURCE]; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + // Tag-delimited aggregate. + // Group type is deprecated and not supported after google.protobuf. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. In Editions, the group wire format + // can be enabled via the `message_encoding` feature. + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + } + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REPEATED = 3; + // The required label is only allowed in google.protobuf. In proto3 and Editions + // it's explicitly prohibited. In Editions, the `field_presence` feature + // can be used to get this behavior. + LABEL_REQUIRED = 2; + } + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + optional string default_value = 7; + + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + + optional FieldOptions options = 8; + + // If true, this is a proto3 "optional". When a proto3 field is optional, it + // tracks presence regardless of field type. + // + // When proto3_optional is true, this field must belong to a oneof to signal + // to old proto3 clients that presence is tracked for this field. This oneof + // is known as a "synthetic" oneof, and this field must be its sole member + // (each proto3 optional field gets its own synthetic oneof). Synthetic oneofs + // exist in the descriptor only, and do not generate any API. Synthetic oneofs + // must be ordered after all "real" oneofs. + // + // For message fields, proto3_optional doesn't create any semantic change, + // since non-repeated message fields always track presence. However it still + // indicates the semantic detail of whether the user wrote "optional" or not. + // This can be useful for round-tripping the .proto file. For consistency we + // give message fields a synthetic oneof also, even though it is not required + // to track presence. This is especially important because the parser can't + // tell if a field is a message or an enum, so it must always create a + // synthetic oneof. + // + // Proto2 optional fields do not set this flag, because they already indicate + // optional with `LABEL_OPTIONAL`. + optional bool proto3_optional = 17; +} + +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default = false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default = false]; +} + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + // Controls the name of the wrapper Java class generated for the .proto file. + // That class will always contain the .proto file's getDescriptor() method as + // well as any top-level extensions defined in the .proto file. + // If java_multiple_files is disabled, then all the other classes from the + // .proto file will be nested inside the single wrapper outer class. + optional string java_outer_classname = 8; + + // If enabled, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the wrapper class + // named by java_outer_classname. However, the wrapper class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default = false]; + + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // A proto2 file can set this to true to opt in to UTF-8 checking for Java, + // which will throw an exception if invalid UTF-8 is parsed from the wire or + // assigned to a string field. + // + // TODO: clarify exactly what kinds of field types this option + // applies to, and update these docs accordingly. + // + // Proto3 files already perform these checks. Setting the option explicitly to + // false has no effect: it cannot be used to opt proto3 files out of UTF-8 + // checks. + optional bool java_string_check_utf8 = 27 [default = false]; + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default = SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. + optional string go_package = 11; + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of google.protobuf. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + reserved 42; // removed php_generic_services + reserved "php_generic_services"; + + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default = false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default = true]; + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // Use this option to change the namespace of php generated metadata classes. + // Default is empty. When this option is empty, the proto file name will be + // used for determining the namespace. + optional string php_metadata_namespace = 44; + + // Use this option to change the package of ruby generated classes. Default + // is empty. When this option is not set, the package name will be used for + // determining the ruby package. + optional string ruby_package = 45; + + // Any features defined in the specific edition. + optional FeatureSet features = 50; + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. + extensions 1000 to max; + + reserved 38; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default = false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default = false]; + + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default = false]; + + reserved 4, 5, 6; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementations still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + + // Enable the legacy handling of JSON field name conflicts. This lowercases + // and strips underscored from the fields before comparison in proto3 only. + // The new behavior takes `json_name` into account and applies to proto2 as + // well. + // + // This should only be used as a temporary measure against broken builds due + // to the change in behavior for JSON field name conflicts. + // + // TODO This is legacy behavior we plan to remove once downstream + // teams have had time to migrate. + optional bool deprecated_legacy_json_field_conflicts = 11 [deprecated = true]; + + // Any features defined in the specific edition. + optional FeatureSet features = 12; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // NOTE: ctype is deprecated. Use `features.(pb.cpp).string_type` instead. + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is only implemented to support use of + // [ctype=CORD] and [ctype=STRING] (the default) on non-repeated fields of + // type "bytes" in the open source release. + // TODO: make ctype actually deprecated. + optional CType ctype = 1 [/*deprecated = true,*/ default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + // The option [ctype=CORD] may be applied to a non-repeated field of type + // "bytes". It indicates that in C++, the data should be stored in a Cord + // instead of a string. For very large strings, this may reduce memory + // fragmentation. It may also allow better performance when parsing from a + // Cord, or when parsing with aliasing enabled, as the parsed Cord may then + // alias the original buffer. + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. This option is prohibited in + // Editions, but the `repeated_field_encoding` feature can be used to control + // the behavior. + optional bool packed = 2; + + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // Note that lazy message fields are still eagerly verified to check + // ill-formed wireformat or missing required fields. Calling IsInitialized() + // on the outer message would fail if the inner message has missing required + // fields. Failed verification would result in parsing failure (except when + // uninitialized messages are acceptable). + optional bool lazy = 5 [default = false]; + + // unverified_lazy does no correctness checks on the byte stream. This should + // only be used where lazy with verification is prohibitive for performance + // reasons. + optional bool unverified_lazy = 15 [default = false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default = false]; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default = false]; + + // Indicate that the field value should not be printed out when using debug + // formats, e.g. when the field contains sensitive credentials. + optional bool debug_redact = 16 [default = false]; + + // If set to RETENTION_SOURCE, the option will be omitted from the binary. + // Note: as of January 2023, support for this is in progress and does not yet + // have an effect (b/264593489). + enum OptionRetention { + RETENTION_UNKNOWN = 0; + RETENTION_RUNTIME = 1; + RETENTION_SOURCE = 2; + } + + optional OptionRetention retention = 17; + + // This indicates the types of entities that the field may apply to when used + // as an option. If it is unset, then the field may be freely used as an + // option on any kind of entity. Note: as of January 2023, support for this is + // in progress and does not yet have an effect (b/264593489). + enum OptionTargetType { + TARGET_TYPE_UNKNOWN = 0; + TARGET_TYPE_FILE = 1; + TARGET_TYPE_EXTENSION_RANGE = 2; + TARGET_TYPE_MESSAGE = 3; + TARGET_TYPE_FIELD = 4; + TARGET_TYPE_ONEOF = 5; + TARGET_TYPE_ENUM = 6; + TARGET_TYPE_ENUM_ENTRY = 7; + TARGET_TYPE_SERVICE = 8; + TARGET_TYPE_METHOD = 9; + } + + repeated OptionTargetType targets = 19; + + message EditionDefault { + optional Edition edition = 3; + optional string value = 2; // Textproto value. + } + repeated EditionDefault edition_defaults = 20; + + // Any features defined in the specific edition. + optional FeatureSet features = 21; + + // Information about the support window of a feature. + message FeatureSupport { + // The edition that this feature was first available in. In editions + // earlier than this one, the default assigned to EDITION_LEGACY will be + // used, and proto files will not be able to override it. + optional Edition edition_introduced = 1; + + // The edition this feature becomes deprecated in. Using this after this + // edition may trigger warnings. + optional Edition edition_deprecated = 2; + + // The deprecation warning text if this feature is used after the edition it + // was marked deprecated in. + optional string deprecation_warning = 3; + + // The edition this feature is no longer available in. In editions after + // this one, the last default assigned will be used, and proto files will + // not be able to override it. + optional Edition edition_removed = 4; + } + optional FeatureSupport feature_support = 22; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype + reserved 18; // reserve target, target_obsolete_do_not_use +} + +message OneofOptions { + // Any features defined in the specific edition. + optional FeatureSet features = 1; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to true to allow mapping different tag names to the same + // value. + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default = false]; + + reserved 5; // javanano_as_lite + + // Enable the legacy handling of JSON field name conflicts. This lowercases + // and strips underscored from the fields before comparison in proto3 only. + // The new behavior takes `json_name` into account and applies to proto2 as + // well. + // TODO Remove this legacy behavior once downstream teams have + // had time to migrate. + optional bool deprecated_legacy_json_field_conflicts = 6 [deprecated = true]; + + // Any features defined in the specific edition. + optional FeatureSet features = 7; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default = false]; + + // Any features defined in the specific edition. + optional FeatureSet features = 2; + + // Indicate that fields annotated with this enum value should not be printed + // out when using debug formats, e.g. when the field contains sensitive + // credentials. + optional bool debug_redact = 3 [default = false]; + + // Information about the support window of a feature value. + optional FieldOptions.FeatureSupport feature_support = 4; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Any features defined in the specific edition. + optional FeatureSet features = 34; + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default = false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default = false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = 34 + [default = IDEMPOTENCY_UNKNOWN]; + + // Any features defined in the specific edition. + optional FeatureSet features = 35; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["moo", false] } represents + // "foo.(bar.baz).moo". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Features + +// TODO Enums in C++ gencode (and potentially other languages) are +// not well scoped. This means that each of the feature enums below can clash +// with each other. The short names we've chosen maximize call-site +// readability, but leave us very open to this scenario. A future feature will +// be designed and implemented to handle this, hopefully before we ever hit a +// conflict here. +message FeatureSet { + enum FieldPresence { + FIELD_PRESENCE_UNKNOWN = 0; + EXPLICIT = 1; + IMPLICIT = 2; + LEGACY_REQUIRED = 3; + } + optional FieldPresence field_presence = 1 [ + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE, + feature_support = { + edition_introduced: EDITION_2023, + }, + edition_defaults = { edition: EDITION_LEGACY, value: "EXPLICIT" }, + edition_defaults = { edition: EDITION_PROTO3, value: "IMPLICIT" }, + edition_defaults = { edition: EDITION_2023, value: "EXPLICIT" } + ]; + + enum EnumType { + ENUM_TYPE_UNKNOWN = 0; + OPEN = 1; + CLOSED = 2; + } + optional EnumType enum_type = 2 [ + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE, + feature_support = { + edition_introduced: EDITION_2023, + }, + edition_defaults = { edition: EDITION_LEGACY, value: "CLOSED" }, + edition_defaults = { edition: EDITION_PROTO3, value: "OPEN" } + ]; + + enum RepeatedFieldEncoding { + REPEATED_FIELD_ENCODING_UNKNOWN = 0; + PACKED = 1; + EXPANDED = 2; + } + optional RepeatedFieldEncoding repeated_field_encoding = 3 [ + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE, + feature_support = { + edition_introduced: EDITION_2023, + }, + edition_defaults = { edition: EDITION_LEGACY, value: "EXPANDED" }, + edition_defaults = { edition: EDITION_PROTO3, value: "PACKED" } + ]; + + enum Utf8Validation { + UTF8_VALIDATION_UNKNOWN = 0; + VERIFY = 2; + NONE = 3; + reserved 1; + } + optional Utf8Validation utf8_validation = 4 [ + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE, + feature_support = { + edition_introduced: EDITION_2023, + }, + edition_defaults = { edition: EDITION_LEGACY, value: "NONE" }, + edition_defaults = { edition: EDITION_PROTO3, value: "VERIFY" } + ]; + + enum MessageEncoding { + MESSAGE_ENCODING_UNKNOWN = 0; + LENGTH_PREFIXED = 1; + DELIMITED = 2; + } + optional MessageEncoding message_encoding = 5 [ + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_FIELD, + targets = TARGET_TYPE_FILE, + feature_support = { + edition_introduced: EDITION_2023, + }, + edition_defaults = { edition: EDITION_LEGACY, value: "LENGTH_PREFIXED" } + ]; + + enum JsonFormat { + JSON_FORMAT_UNKNOWN = 0; + ALLOW = 1; + LEGACY_BEST_EFFORT = 2; + } + optional JsonFormat json_format = 6 [ + retention = RETENTION_RUNTIME, + targets = TARGET_TYPE_MESSAGE, + targets = TARGET_TYPE_ENUM, + targets = TARGET_TYPE_FILE, + feature_support = { + edition_introduced: EDITION_2023, + }, + edition_defaults = { edition: EDITION_LEGACY, value: "LEGACY_BEST_EFFORT" }, + edition_defaults = { edition: EDITION_PROTO3, value: "ALLOW" } + ]; + + reserved 999; + + extensions 1000 to 9994 [ + declaration = { + number: 1000, + full_name: ".pb.cpp", + type: ".pb.CppFeatures" + }, + declaration = { + number: 1001, + full_name: ".pb.java", + type: ".pb.JavaFeatures" + }, + declaration = { number: 1002, full_name: ".pb.go", type: ".pb.GoFeatures" }, + declaration = { + number: 9990, + full_name: ".pb.proto1", + type: ".pb.Proto1Features" + } + ]; + + extensions 9995 to 9999; // For internal testing + extensions 10000; // for https://github.com/bufbuild/protobuf-es +} + +// A compiled specification for the defaults of a set of features. These +// messages are generated from FeatureSet extensions and can be used to seed +// feature resolution. The resolution with this object becomes a simple search +// for the closest matching edition, followed by proto merges. +message FeatureSetDefaults { + // A map from every known edition with a unique set of defaults to its + // defaults. Not all editions may be contained here. For a given edition, + // the defaults at the closest matching edition ordered at or before it should + // be used. This field must be in strict ascending order by edition. + message FeatureSetEditionDefault { + optional Edition edition = 3; + + // Defaults of features that can be overridden in this edition. + optional FeatureSet overridable_features = 4; + + // Defaults of features that can't be overridden in this edition. + optional FeatureSet fixed_features = 5; + + reserved 1, 2; + reserved "features"; + } + repeated FeatureSetEditionDefault defaults = 1; + + // The minimum supported edition (inclusive) when this was constructed. + // Editions before this will not have defaults. + optional Edition minimum_edition = 4; + + // The maximum known edition (inclusive) when this was constructed. Editions + // after this will not have reliable defaults. + optional Edition maximum_edition = 5; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendant. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition appears. + // For example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed = true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed = true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to moo. + // // + // // Another line attached to moo. + // optional double moo = 4; + // + // // Detached comment for corge. This is not leading or trailing comments + // // to moo or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + // + // // ignored detached comments. + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed = true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified object. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; + + // Represents the identified object's effect on the element in the original + // .proto file. + enum Semantic { + // There is no effect or the effect is indescribable. + NONE = 0; + // The element is set or otherwise mutated. + SET = 1; + // An alias to the element is returned. + ALIAS = 2; + } + optional Semantic semantic = 5; + } +} diff --git a/proto-lens.cabal b/proto-lens.cabal --- a/proto-lens.cabal +++ b/proto-lens.cabal @@ -1,45 +1,107 @@ -name: proto-lens -version: 0.2.2.0 -synopsis: A lens-based implementation of protocol buffers in Haskell. -description: - The proto-lens library provides to protocol buffers using modern - Haskell language and library patterns. Specifically, it provides: - . - * Composable field accessors via lenses - . - * Simple field name resolution/overloading via type-level literals - . - * Type-safe reflection and encoding/decoding of messages via GADTs +cabal-version: 1.12 -homepage: https://github.com/google/proto-lens -license: BSD3 -license-file: LICENSE -author: Judah Jacobson -maintainer: proto-lens@googlegroups.com -copyright: Google Inc. -category: Data -build-type: Simple -cabal-version: >=1.8 -extra-source-files: Changelog.md +-- This file has been generated from package.yaml by hpack version 0.39.1. +-- +-- see: https://github.com/sol/hpack +name: proto-lens +version: 0.7.1.7 +synopsis: A lens-based implementation of protocol buffers in Haskell. +description: The proto-lens library provides an API for protocol buffers using modern Haskell language and library patterns. Specifically, it provides: + . + * Composable field accessors via lenses + . + * Simple field name resolution/overloading via type-level literals + . + * Type-safe reflection and encoding/decoding of messages via GADTs +category: Data +homepage: https://github.com/google/proto-lens#readme +bug-reports: https://github.com/google/proto-lens/issues +author: Judah Jacobson +maintainer: proto-lens@googlegroups.com +copyright: Google Inc. +license: BSD3 +license-file: LICENSE +build-type: Simple +extra-source-files: + Changelog.md +data-files: + proto-lens-imports/google/protobuf/descriptor.proto + proto-lens-imports/google/protobuf/compiler/plugin.proto + +source-repository head + type: git + location: https://github.com/google/proto-lens + subdir: proto-lens + library - hs-source-dirs: src - exposed-modules: Data.ProtoLens - Data.ProtoLens.Encoding - Data.ProtoLens.Message - Data.ProtoLens.Message.Enum - Data.ProtoLens.TextFormat - other-modules: Data.ProtoLens.Encoding.Bytes - Data.ProtoLens.Encoding.Wire - Data.ProtoLens.TextFormat.Parser - build-depends: attoparsec == 0.13.* - , base >= 4.8 && < 4.11 - , bytestring == 0.10.* - , containers == 0.5.* - , data-default-class >= 0.0 && < 0.2 - , lens-family == 1.2.* - , parsec == 3.1.* - , pretty == 1.1.* - , text == 1.2.* - , transformers >= 0.4 && < 0.6 - , void == 0.7.* + exposed-modules: + Data.ProtoLens + Data.ProtoLens.Combinators + Data.ProtoLens.Default + Data.ProtoLens.Encoding + Data.ProtoLens.Encoding.Bytes + Data.ProtoLens.Encoding.Growing + Data.ProtoLens.Encoding.Parser + Data.ProtoLens.Encoding.Parser.Unsafe + Data.ProtoLens.Encoding.Wire + Data.ProtoLens.Field + Data.ProtoLens.Labels + Data.ProtoLens.Message + Data.ProtoLens.Message.Enum + Data.ProtoLens.Prism + Data.ProtoLens.Service.Types + Data.ProtoLens.TextFormat + other-modules: + Data.ProtoLens.Encoding.Parser.Internal + Data.ProtoLens.TextFormat.Parser + hs-source-dirs: + src + build-depends: + base >=4.10 && <4.23 + , bytestring >=0.10 && <0.14 + , containers >=0.5 && <0.9 + , deepseq >=1.4 && <1.6 + , ghc-prim >=0.4 && <0.14 + , lens-family >=1.2 && <2.2 + , parsec ==3.1.* + , pretty ==1.1.* + , primitive >=0.6 && <0.10 + , profunctors >=5.2 && <6.0 + , tagged ==0.8.* + , text >=1.2 && <2.2 + , transformers >=0.4 && <0.7 + , vector >=0.11 && <0.14 + default-language: Haskell2010 + +test-suite growing_test + type: exitcode-stdio-1.0 + main-is: growing_test.hs + other-modules: + Paths_proto_lens + hs-source-dirs: + tests + build-depends: + QuickCheck + , base + , proto-lens + , tasty + , tasty-quickcheck + , vector + default-language: Haskell2010 + +test-suite parser_test + type: exitcode-stdio-1.0 + main-is: parser_test.hs + other-modules: + Paths_proto_lens + hs-source-dirs: + tests + build-depends: + QuickCheck + , base + , bytestring + , proto-lens + , tasty + , tasty-quickcheck + default-language: Haskell2010 diff --git a/src/Data/ProtoLens/Combinators.hs b/src/Data/ProtoLens/Combinators.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Combinators.hs @@ -0,0 +1,47 @@ +-- Copyright 2016 Google Inc. All Rights Reserved. +-- +-- Use of this source code is governed by a BSD-style +-- license that can be found in the LICENSE file or at +-- https://developers.google.com/open-source/licenses/bsd + +{-# LANGUAGE RankNTypes #-} +-- | An assorted collection of functions useful when working with ProtoLens +-- protocol buffers. These functions are inspired by functionality found in +-- the protobuf implementation in other languages. +module Data.ProtoLens.Combinators + ( has + , clear + , make + , modifyInState + ) where + +import Control.Monad.Trans.State (State, execState) +import Data.ProtoLens (Message(..)) +import Data.Maybe (isJust) +import Lens.Family2 (LensLike, Phantom, Setter, to, (.~)) + +-- | Turns a 'LensLike' getting a 'Maybe' into a 'Getter' of a 'Bool' +-- that returns @True@ when the 'Maybe' is @Just@ something. +-- +-- I.e., makes a getter for a 'Maybe' field that returns whether it's set. +has :: Phantom f => LensLike f a a' (Maybe b) b' -> LensLike f a a' Bool b'' +has = (. to isJust) + +-- | Sets the provided lens in @a@ to @Nothing@. +clear :: Setter a a' b (Maybe b') -> a -> a' +clear setter = setter .~ Nothing + +-- | Creates a 'Default' and then applies the provided `State` to it. This is +-- convenient for creating complicated structures. +make :: Message msg => State msg a -> msg +make = modifyInState defMessage + +-- | Allows one to modify a value in the 'State' monad. Note that this is +-- just for syntactic convenience with @do@ blocks, e.g. +-- +-- @ +-- newThing = modifyInState thing $ do +-- ... +-- @ +modifyInState :: s -> State s a -> s +modifyInState = flip execState diff --git a/src/Data/ProtoLens/Default.hs b/src/Data/ProtoLens/Default.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Default.hs @@ -0,0 +1,20 @@ +-- | A compatibility layer for older code to create default protocol buffer messages. +-- +-- In older versions of @proto-lens@, messages could be constructed with +-- @Data.Default.Class.def@. However, for @proto-lens >= 0.4@, that is +-- no longer the case and @Data.ProtoLens.defMessage@ should be used instead. +-- +-- This module provides a compatibility layer that may be used to upgrade +-- older code without substantial code changes. +module Data.ProtoLens.Default + ( def + , Message + ) where + +import Data.ProtoLens.Message (Message(defMessage)) + +-- | A message with all fields set to their default values. +-- +-- For new code, prefer `defMessage`. +def :: Message a => a +def = defMessage diff --git a/src/Data/ProtoLens/Encoding.hs b/src/Data/ProtoLens/Encoding.hs --- a/src/Data/ProtoLens/Encoding.hs +++ b/src/Data/ProtoLens/Encoding.hs @@ -1,73 +1,33 @@ --- Copyright 2016 Google Inc. All Rights Reserved. --- --- Use of this source code is governed by a BSD-style --- license that can be found in the LICENSE file or at --- https://developers.google.com/open-source/licenses/bsd - --- | Functions for encoding and decoding protocol buffer Messages. --- --- TODO: Currently all operations are on strict ByteStrings; --- we should try to generalize to lazy Bytestrings as well. -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE GADTs #-} -{-# LANGUAGE LambdaCase #-} -{-# LANGUAGE PatternGuards #-} -{-# LANGUAGE ScopedTypeVariables #-} -module Data.ProtoLens.Encoding( +{-# LANGUAGE CPP #-} +module Data.ProtoLens.Encoding ( encodeMessage, buildMessage, decodeMessage, + parseMessage, decodeMessageOrDie, + -- ** Delimited messages + buildMessageDelimited, + parseMessageDelimited, + decodeMessageDelimitedH, ) where -import Data.ProtoLens.Message -import Data.ProtoLens.Encoding.Bytes -import Data.ProtoLens.Encoding.Wire +import System.IO (Handle) -import Control.Applicative ((<|>)) -import Control.Monad (guard) -import Data.Attoparsec.ByteString as Parse -import Data.Bool (bool) -import Data.Text.Encoding (encodeUtf8, decodeUtf8') -import Data.Text.Encoding.Error (UnicodeException(..)) -import qualified Data.ByteString as B -import qualified Data.Map.Strict as Map -import Data.ByteString.Lazy.Builder as Builder -import qualified Data.ByteString.Lazy as L -import Lens.Family2 (set, over, (^.), (&)) +import Data.ProtoLens.Message (Message(..)) +import Data.ProtoLens.Encoding.Bytes (Parser, Builder) +import qualified Data.ProtoLens.Encoding.Bytes as Bytes --- TODO: We could be more incremental when parsing/encoding length-based fields, --- rather than forcing the whole thing. E.g., for encoding we're doing extra --- allocation by building an intermediate bytestring. +import Control.Monad.IO.Class (liftIO) +import Control.Monad.Trans.Except (runExceptT, ExceptT(..)) +import qualified Data.ByteString as B +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup ((<>)) +#endif -- | Decode a message from its wire format. Returns 'Left' if the decoding -- fails. decodeMessage :: Message msg => B.ByteString -> Either String msg -decodeMessage input = parseOnly (parseMessage endOfInput) input - --- | Parse a message with the given ending delimiter (which will be EndGroup in --- the case of a group, and end-of-input otherwise). -parseMessage :: forall msg . Message msg => Parser () -> Parser msg -parseMessage end = do - (msg, unsetFields) <- loop def requiredFields - if Map.null unsetFields - then return $ reverseRepeatedFields fields msg - else fail $ "Missing required fields " - ++ show (map fieldDescriptorName - $ Map.elems $ unsetFields) - where - fields = fieldsByTag descriptor - requiredFields = Map.filter isRequired fields - loop :: msg -> Map.Map Tag (FieldDescriptor msg) - -> Parser (msg, Map.Map Tag (FieldDescriptor msg)) - loop msg unsetFields = ((msg, unsetFields) <$ end) - <|> do - tv@(TaggedValue tag _) <- getTaggedValue - case Map.lookup (Tag tag) fields of - Nothing -> loop msg unsetFields - Just field -> do - !msg' <- parseAndAddField msg field tv - loop msg' $! Map.delete (Tag tag) unsetFields +decodeMessage = Bytes.runParser parseMessage -- | Decode a message from its wire format. Throws an error if the decoding -- fails. @@ -76,178 +36,29 @@ Left e -> error $ "decodeMessageOrDie: " ++ e Right x -> x -runEither :: Either String a -> Parser a -runEither = either fail return - -parseAndAddField :: msg - -> FieldDescriptor msg - -> TaggedValue - -> Parser msg -parseAndAddField - !msg - (FieldDescriptor name typeDescriptor accessor) - (TaggedValue tag (WireValue wt val)) = let - getSimpleVal = case fieldWireType typeDescriptor of - GroupFieldType -> do - Equal <- equalWireTypes name StartGroup wt - parseMessage (endOfGroup name tag) - FieldWireType fieldWt _ get -> do - Equal <- equalWireTypes name fieldWt wt - runEither $ get val - -- Get a block of packed values, reversed. - getPackedVals = case fieldWireType typeDescriptor of - GroupFieldType -> fail "Groups can't be packed" - FieldWireType fieldWt _ get -> do - Equal <- equalWireTypes name Lengthy wt - let getElt = do - wv <- getWireValue fieldWt tag - x <- runEither $ get wv - return $! x - runEither $ parseOnly (manyReversedTill getElt endOfInput) val - in case accessor of - PlainField _ f -> do - !x <- getSimpleVal - return $! set f x msg - OptionalField f -> do - !x <- getSimpleVal - return $! set f (Just x) msg - -- Parse either a packed or unpacked representation, - -- depending on how it was encoded. - -- Note that if fieldWt is Lengthy (e.g., "string" or - -- message) we should always parse it as unpacked. - RepeatedField _ f - -> (do - !x <- getSimpleVal - return $! over f (\(!xs) -> x:xs) msg) - <|> (do - xs <- getPackedVals - return $! over f (\(!ys) -> xs++ys) msg) - <|> fail ("Field " ++ name - ++ "expects a repeated field wire type but found " - ++ show wt) - MapField keyLens valueLens f -> do - entry <- getSimpleVal - let !key = entry ^. keyLens - let !value = entry ^. valueLens - return $! over f - (Map.insert key value) - msg - --- | Run the parser zero or more times, until the "end" parser succeeds. --- Returns a list of the parsed elements, in reverse order. -manyReversedTill :: Parser a -> Parser b -> Parser [a] -manyReversedTill p end = loop [] - where - loop xs = (end >> return xs) <|> (p >>= \x -> loop (x:xs)) - -- | Encode a message to the wire format as a strict 'ByteString'. encodeMessage :: Message msg => msg -> B.ByteString -encodeMessage = L.toStrict . toLazyByteString . buildMessage - --- | Encode a message to the wire format, as part of a 'Builder'. -buildMessage :: Message msg => msg -> Builder -buildMessage msg = foldMap putTaggedValue (messageToTaggedValues msg) - --- | Encode a message as a sequence of key-value pairs. -messageToTaggedValues :: Message msg => msg -> [TaggedValue] -messageToTaggedValues msg = mconcat - [ messageFieldToVals t fieldDescr msg - | (Tag t, fieldDescr) <- Map.toList (fieldsByTag descriptor) - ] - -messageFieldToVals :: Int -> FieldDescriptor a -> a -> [TaggedValue] -messageFieldToVals tag (FieldDescriptor _ typeDescriptor accessor) msg = - let - embed src - = case fieldWireType typeDescriptor of - FieldWireType wt convert _ - -> [TaggedValue tag $ WireValue wt (convert src)] - GroupFieldType - -> TaggedValue tag (WireValue StartGroup ()) - : messageToTaggedValues src - ++ [TaggedValue tag $ WireValue EndGroup ()] - embedPacked [] = [] - embedPacked src - = case fieldWireType typeDescriptor of - GroupFieldType -> error "GroupFieldType can't be packed" - FieldWireType wt convert _ -> let - v = L.toStrict $ toLazyByteString - $ mconcat [putWireValue wt (convert x) | x <- src] - in [TaggedValue tag $ WireValue Lengthy v] - in case accessor of - PlainField d f - -- proto3 optional scalar field: - | Optional <- d, src == fieldDefault -> [] - -- proto3 optional non-scalar field, or proto2 required field: - | otherwise -> embed src - where src = msg ^. f - -- proto2 optional field: - OptionalField f -> foldMap embed (msg ^. f) - -- Note: using 'concatMap' instead of 'foldMap' below - -- seems to allow better list fusion. - RepeatedField Unpacked f -> concatMap embed (msg ^. f) - RepeatedField Packed f -> embedPacked (msg ^. f) - MapField keyLens valueLens f -> - concatMap (\(k, v) -> embed $ def & set keyLens k & set valueLens v) - $ Map.toList (msg ^. f) - -data FieldWireType value where - FieldWireType :: WireType w -> (value -> w) -> (w -> Either String value) - -> FieldWireType value - GroupFieldType :: Message value => FieldWireType value - -fieldWireType :: FieldTypeDescriptor value -> FieldWireType value --- TODO: Don't let toEnum crash on unknown enum values. -fieldWireType EnumField = simpleFieldWireType VarInt - (fromIntegral . fromEnum) - (toEnum . fromIntegral) -fieldWireType BoolField = simpleFieldWireType VarInt (bool 0 1) (/= 0) --- Note: int{32,64} and sfixed{32,64} are stored using the signed -> unsigned --- conversion of fromIntegral. -fieldWireType Int32Field = integralFieldWireType VarInt -fieldWireType Int64Field = integralFieldWireType VarInt -fieldWireType UInt32Field = integralFieldWireType VarInt -fieldWireType UInt64Field = identityFieldWireType VarInt -fieldWireType SInt32Field = simpleFieldWireType VarInt - (fromIntegral . signedInt32ToWord) - (wordToSignedInt32 . fromIntegral) -fieldWireType SInt64Field = simpleFieldWireType VarInt - signedInt64ToWord wordToSignedInt64 -fieldWireType Fixed32Field = identityFieldWireType Fixed32 -fieldWireType Fixed64Field = identityFieldWireType Fixed64 -fieldWireType SFixed32Field = integralFieldWireType Fixed32 -fieldWireType SFixed64Field = integralFieldWireType Fixed64 -fieldWireType FloatField = simpleFieldWireType Fixed32 floatToWord wordToFloat -fieldWireType DoubleField = simpleFieldWireType Fixed64 - doubleToWord wordToDouble -fieldWireType StringField = FieldWireType Lengthy encodeUtf8 - (stringizeError . decodeUtf8') -fieldWireType BytesField = identityFieldWireType Lengthy -fieldWireType MessageField = FieldWireType Lengthy encodeMessage - decodeMessage -fieldWireType GroupField = GroupFieldType - -endOfGroup :: String -> Int -> Parser () -endOfGroup name tag = do - TaggedValue tag' (WireValue wt _) <- getTaggedValue - Equal <- equalWireTypes name EndGroup wt - guard (tag == tag') - --- | Helper function to define a field type whose decoding operation can't fail. -simpleFieldWireType :: WireType w -> (value -> w) -> (w -> value) - -> FieldWireType value -simpleFieldWireType w f g = FieldWireType w f (return . g) +encodeMessage = Bytes.runBuilder . buildMessage --- | A simple field type which is the same as its wire type. -identityFieldWireType :: WireType w -> FieldWireType w -identityFieldWireType w = simpleFieldWireType w id id +-- | Encode a message to the wire format, prefixed by its size as a VarInt, +-- as part of a 'Builder'. +-- +-- This can be used to build up streams of messages in the size-delimited +-- format expected by some protocols. +buildMessageDelimited :: Message msg => msg -> Builder +buildMessageDelimited msg = + let b = encodeMessage msg + in Bytes.putVarInt (fromIntegral $ B.length b) <> Bytes.putBytes b --- | A simple field type which converts to/from its wire type via --- "fromIntegral". -integralFieldWireType - :: (Integral w, Integral value) => WireType w -> FieldWireType value -integralFieldWireType w = simpleFieldWireType w fromIntegral fromIntegral +parseMessageDelimited :: Message msg => Parser msg +parseMessageDelimited = do + len <- Bytes.getVarInt + bytes <- Bytes.getBytes $ fromIntegral len + either fail return $ decodeMessage bytes -stringizeError :: Either UnicodeException a -> Either String a -stringizeError (Left e) = Left (show e) -stringizeError (Right a) = Right a +-- | Same as @decodeMessage@ but for delimited messages read through a Handle +decodeMessageDelimitedH :: Message msg => Handle -> IO (Either String msg) +decodeMessageDelimitedH h = runExceptT $ + Bytes.getVarIntH h >>= + liftIO . B.hGet h . fromIntegral >>= + ExceptT . return . decodeMessage diff --git a/src/Data/ProtoLens/Encoding/Bytes.hs b/src/Data/ProtoLens/Encoding/Bytes.hs --- a/src/Data/ProtoLens/Encoding/Bytes.hs +++ b/src/Data/ProtoLens/Encoding/Bytes.hs @@ -5,64 +5,169 @@ -- https://developers.google.com/open-source/licenses/bsd {-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE LambdaCase #-} -- | Utility functions for parsing and encoding individual types. module Data.ProtoLens.Encoding.Bytes( + -- * Running encodings + Parser, + Builder, + runParser, + isolate, + runBuilder, + -- * Bytestrings + getBytes, + putBytes, + -- * Text + getText, + -- * Integral types getVarInt, + getVarIntH, putVarInt, - anyBits, + getFixed32, + getFixed64, + putFixed32, + putFixed64, + -- * Floating-point types wordToFloat, wordToDouble, floatToWord, doubleToWord, + -- * Signed types signedInt32ToWord, wordToSignedInt32, signedInt64ToWord, wordToSignedInt64, + -- * Other utilities + atEnd, + runEither, + (), + foldMapBuilder, ) where -import Data.Attoparsec.ByteString as Parse +import Control.Monad.IO.Class (liftIO) +import Control.Monad.Trans.Except (throwE, ExceptT) import Data.Bits -import Data.ByteString.Lazy.Builder as Builder +import Data.ByteString (ByteString) +import Data.ByteString.Builder as Builder +import qualified Data.ByteString.Builder.Internal as Internal +import qualified Data.ByteString.Lazy as L import Data.Int (Int32, Int64) -import Data.Monoid ((<>)) -import Data.Word (Word32, Word64) +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup ((<>)) +#endif +import qualified Data.Vector.Generic as V +import Data.Word (Word8, Word32, Word64) +import Foreign.Marshal (malloc, free) +import System.IO (Handle, hGetBuf) +#if MIN_VERSION_base(4,11,0) +import qualified GHC.Float as Float +import Foreign.Storable (peek) +#else import Foreign.Ptr (castPtr) import Foreign.Marshal.Alloc (alloca) import Foreign.Storable (Storable, peek, poke) import System.IO.Unsafe (unsafePerformIO) +#endif +import Data.ProtoLens.Encoding.Parser + +-- | Constructs a strict 'ByteString' from the given 'Builder'. +runBuilder :: Builder -> ByteString +runBuilder = L.toStrict . Builder.toLazyByteString + +-- | Emit a given @ByteString@. +putBytes :: ByteString -> Builder +putBytes = Builder.byteString + -- VarInts are inherently unsigned; there are different ways of encoding -- negative numbers for int32/64 and sint32/64. getVarInt :: Parser Word64 -getVarInt = loop 1 0 +getVarInt = loopStart 0 1 where - loop !s !n = do - b <- anyWord8 - let n' = n + s * fromIntegral (b .&. 127) - if (b .&. 128) == 0 - then return $! n' - else loop (128*s) n' + loopStart !n !s = getWord8 >>= getVarIntLoopFinish loopStart n s --- | Little-endian decoding function. -anyBits :: forall a . (Num a, FiniteBits a) => Parser a -anyBits = loop 0 0 - where - size = finiteBitSize (undefined :: a) - loop !w !n - | n >= size = return w - | otherwise = do - b <- anyWord8 - loop (w .|. shiftL (fromIntegral b) n) (n+8) +-- Same as getVarInt but reads from a Handle +getVarIntH :: Handle -> ExceptT String IO Word64 +getVarIntH h = do + buf <- liftIO malloc + let loopStart !n !s = + liftIO (hGetBuf h buf 1) >>= + \case + 1 -> liftIO (peek buf) >>= + getVarIntLoopFinish loopStart n s + _ -> throwE "Unexpected end of file" + res <- loopStart 0 1 + liftIO $ free buf + return res +getVarIntLoopFinish + :: (Monad m) + => (Word64 -> Word64 -> m Word64) -- "loop start" callback + -> Word64 + -> Word64 + -> Word8 + -> m Word64 +getVarIntLoopFinish ls !n !s !b = do + let n' = decodeVarIntStep n s b + if testMsb b + then ls n' (128*s) + else return $! n' + +-- n -- result of previous step; s -- 128^{step index}; b -- step byte +decodeVarIntStep :: Word64 -> Word64 -> Word8 -> Word64 +decodeVarIntStep n s b = n + s * fromIntegral (b .&. 127) + +testMsb :: Word8 -> Bool +testMsb b = (b .&. 128) /= 0 + putVarInt :: Word64 -> Builder putVarInt n | n < 128 = Builder.word8 (fromIntegral n) - | otherwise = Builder.word8 (fromIntegral $ n .&. 127 .|. 128) + | otherwise = Builder.word8 (fromIntegral $ n .|. 128) <> putVarInt (n `shiftR` 7) +getFixed32 :: Parser Word32 +getFixed32 = getWord32le + +getFixed64 :: Parser Word64 +getFixed64 = do + x <- getFixed32 + y <- getFixed32 + return $ fromIntegral y `shiftL` 32 + fromIntegral x + +-- Note: putFixed32 and putFixed32 have added BangPatterns over the +-- standard Builders. +-- This works better when they're composed with other functions. +-- For example, consider `putFixed32 . floatToWord`. +-- Since `putFixed32` may return a continuation, it doesn't automatically +-- force the result of `floatToWord`, so the resulting Word32 must be kept +-- lazily. The extra strictness means that the Word32 will be evaluated +-- outside of the continuation, and GHC can pass it around unboxed. + +putFixed32 :: Word32 -> Builder +putFixed32 !x = word32LE x + +putFixed64 :: Word64 -> Builder +putFixed64 !x = word64LE x + +#if MIN_VERSION_base(4,11,0) +wordToDouble :: Word64 -> Double +wordToDouble = Float.castWord64ToDouble + +wordToFloat :: Word32 -> Float +wordToFloat = Float.castWord32ToFloat + +doubleToWord :: Double -> Word64 +doubleToWord = Float.castDoubleToWord64 + +floatToWord :: Float -> Word32 +floatToWord = Float.castFloatToWord32 + +#else -- WARNING: SUPER UNSAFE! -- Helper function purely for converting between Word32/Word64 and -- Float/Double. Note that ideally we could just use unsafeCoerce, but this @@ -88,6 +193,7 @@ floatToWord :: Float -> Word32 floatToWord = cast +#endif signedInt32ToWord :: Int32 -> Word32 signedInt32ToWord n = fromIntegral $ shiftL n 1 `xor` shiftR n 31 @@ -102,3 +208,30 @@ wordToSignedInt64 :: Word64 -> Int64 wordToSignedInt64 n = fromIntegral (shiftR n 1) `xor` negate (fromIntegral $ n .&. 1) + +runEither :: Either String a -> Parser a +runEither = either fail return + +-- | Loop over the elements of a vector and concatenate the resulting +-- @Builder@s. +-- +-- This function has been hand-tuned to perform better than a naive +-- implementation using, e.g., Vector.foldr or a manual loop. +foldMapBuilder :: V.Vector v a => (a -> Builder) -> v a -> Builder +foldMapBuilder f = \v0 -> Internal.builder (loop v0) + -- Place v0 on the right-hand side so that GHC actually inlines + -- this function. + where + -- Fully-saturate the inner loop (rather than currying away `cont` + -- and `bs`) to avoid GHC creating an intermediate continuation. + loop v cont bs + | V.null v = cont bs + | otherwise = let + !x = V.unsafeHead v + -- lts-8.24 (ghc-8.0) doesn't inline unsafeTail well. + -- We can remove the following bang when we bump the lower bound: + !xs = V.unsafeTail v + in Internal.runBuilderWith + (f x) + (loop xs cont) bs +{-# INLINE foldMapBuilder #-} diff --git a/src/Data/ProtoLens/Encoding/Growing.hs b/src/Data/ProtoLens/Encoding/Growing.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Encoding/Growing.hs @@ -0,0 +1,70 @@ +-- | A mutable vector that grows in size. +-- +-- Example usage: +-- +-- > import qualified Data.ProtoLens.Encoding.Growing as Growing +-- > import qualified Data.Vector.Unboxed as V +-- > test :: IO (V.Vector Int) +-- > test = do +-- > v <- Growing.new +-- > v' <- Growing.append v 1 +-- > v'' <- Growing.append v' 2 +-- > v''' <- Growing.append v'' 3 +-- > unsafeFreeze v''' +module Data.ProtoLens.Encoding.Growing ( + Growing, + new, + append, + unsafeFreeze, + RealWorld, + ) where + +import Control.Monad.Primitive (PrimMonad, PrimState, RealWorld) + +import qualified Data.Vector.Generic as V +import qualified Data.Vector.Generic.Mutable as MV + +-- | A mutable vector which can increase in capacity. +data Growing v s a = Growing + {-# UNPACK #-} !Int + -- The number of elements in the mutable vector + -- that have already been set. + !(V.Mutable v s a) + -- TODOs for efficiency: + -- - Try unpacking this. It's difficult as-is because + -- V.Mutable is a type function. + -- - MVectors support slicing, but we're not using that + -- functionality, so we're passing around an extra unnecessary + -- Int. + +-- | Create a new empty growing vector. +new :: (PrimMonad m, V.Vector v a) => m (Growing v (PrimState m) a) +new = Growing 0 <$> MV.new 0 + +-- | Unsafely convert a growing vector to an immutable one without +-- copying. After this call, you may not use the growing vector +-- nor any other growing vectors that were used to produce this one. +unsafeFreeze + :: (PrimMonad m, V.Vector v a) + => Growing v (PrimState m) a -> m (v a) +unsafeFreeze (Growing len m) = V.unsafeFreeze (MV.take len m) + +-- | Returns a new growing vector with a new element at the end. +-- Note that the return value may share storage with the input value. +-- Furthermore, calling @append@ twice on the same input may result +-- in two vectors that share the same storage. +append + :: (PrimMonad m, V.Vector v a) + => Growing v (PrimState m) a + -> a + -> m (Growing v (PrimState m) a) +append (Growing len v) x + | len < MV.length v = do + MV.unsafeWrite v len x + return $ Growing (len + 1) v + | otherwise = do + let len' = 2 * len + 1 + v' <- MV.unsafeGrow v len' + MV.unsafeWrite v' len x + return $ Growing (len + 1) v' +{-# INLINE append #-} diff --git a/src/Data/ProtoLens/Encoding/Parser.hs b/src/Data/ProtoLens/Encoding/Parser.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Encoding/Parser.hs @@ -0,0 +1,120 @@ +-- | A custom parsing monad, optimized for speed. +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} +module Data.ProtoLens.Encoding.Parser + ( Parser + , runParser + , atEnd + , isolate + , getWord8 + , getWord32le + , getBytes + , getText + , () + ) where + +import Data.Bits (shiftL, (.|.)) +import Data.Word (Word8, Word32) +import Data.ByteString (ByteString, packCStringLen) +import qualified Data.ByteString.Unsafe as B +import Data.Text (Text) +import Data.Text.Encoding (decodeUtf8') +import Foreign.Ptr +import Foreign.Storable +import System.IO.Unsafe (unsafePerformIO) + +import Data.ProtoLens.Encoding.Parser.Internal + +-- | Evaluates a parser on the given input. +-- +-- If the parser does not consume all of the input, the rest of the +-- input is discarded and the parser still succeeds. Parsers may use +-- 'atEnd' to detect whether they are at the end of the input. +-- +-- Values returned from actions in this monad will not hold onto the original +-- ByteString, but rather make immutable copies of subsets of its bytes. +runParser :: Parser a -> ByteString -> Either String a +runParser (Parser m) b = + case unsafePerformIO $ B.unsafeUseAsCStringLen b + $ \(p, len) -> m (p `plusPtr` len) (castPtr p) of + ParseSuccess _ x -> Right x + ParseFailure s -> Left s + +-- | Returns True if there is no more input left to consume. +atEnd :: Parser Bool +atEnd = Parser $ \end pos -> return $ ParseSuccess pos (pos == end) + +-- | Parse a one-byte word. +getWord8 :: Parser Word8 +getWord8 = withSized 1 "getWord8: Unexpected end of input" peek + +-- | Parser a 4-byte word in little-endian order. +getWord32le :: Parser Word32 +getWord32le = withSized 4 "getWord32le: Unexpected end of input" $ \pos -> do + b1 <- fromIntegral <$> peek pos + b2 <- fromIntegral <$> peek (pos `plusPtr'` 1) + b3 <- fromIntegral <$> peek (pos `plusPtr'` 2) + b4 <- fromIntegral <$> peek (pos `plusPtr'` 3) + let f b b' = b `shiftL` 8 .|. b' + return $! f (f (f b4 b3) b2) b1 + +-- | Parse a sequence of zero or more bytes of the given length. +-- +-- The new ByteString is an immutable copy of the bytes in the input +-- and will be managed separately on the Haskell heap from the original +-- input 'ByteString'. +-- +-- Fails the parse if given a negative length. +getBytes :: Int -> Parser ByteString +getBytes n = withSized n "getBytes: Unexpected end of input" + $ \pos -> packCStringLen (castPtr pos, n) + +getText :: Int -> Parser Text +getText n = do + r <- withSized n "getText: Unexpected end of input" $ \pos -> + decodeUtf8' <$> B.unsafePackCStringLen (castPtr pos, n) + either (fail . show) pure r + +-- | Helper function for reading bytes from the current position and +-- advancing the pointer. +-- +-- Fails the parse if given a negative length. (GHC will elide the check +-- if the length is a nonnegative constant.) +-- +-- It is only safe for @f@ to peek between its argument @p@ and +-- @p `plusPtr` (len - 1)@, inclusive. +withSized :: Int -> String -> (Ptr Word8 -> IO a) -> Parser a +withSized len message f + | len >= 0 = Parser $ \end pos -> + let pos' = pos `plusPtr'` len + in if pos' > end + then return $ ParseFailure message + else ParseSuccess pos' <$> f pos + | otherwise = fail "withSized: negative length" +{-# INLINE withSized #-} + +-- | Run the given parsing action as if there are only +-- @len@ bytes remaining. That is, once @len@ bytes have been +-- consumed, 'atEnd' will return 'True' and other actions +-- like 'getWord8' will act like there is no input remaining. +-- +-- Fails the parse if given a negative length. +isolate :: Int -> Parser a -> Parser a +isolate len (Parser m) + | len >= 0 = Parser $ \end pos -> + let end' = pos `plusPtr` len + in if end' > end + then return $ ParseFailure "isolate: unexpected end of input" + else m end' pos + | otherwise = fail "isolate: negative length" + +-- | If the parser fails, prepend an error message. +() :: Parser a -> String -> Parser a +Parser m msg = Parser $ \end p -> wrap <$> m end p + where + wrap (ParseFailure s) = ParseFailure (msg ++ ": " ++ s) + wrap r = r + +-- | Advance a pointer. Unlike 'plusPtr', preserves the type of the input. +plusPtr' :: Ptr a -> Int -> Ptr a +plusPtr' = plusPtr diff --git a/src/Data/ProtoLens/Encoding/Parser/Internal.hs b/src/Data/ProtoLens/Encoding/Parser/Internal.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Encoding/Parser/Internal.hs @@ -0,0 +1,50 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE LambdaCase #-} +-- | Definition of the parsing monad, plus internal +-- unsafe functions. +module Data.ProtoLens.Encoding.Parser.Internal + ( Parser(..) + , ParseResult(..) + ) where + +import Control.Monad (ap) +import qualified Control.Monad.Fail as Fail +import Data.Word (Word8) +import Foreign.Ptr + +-- | A monad for parsing an input buffer. +newtype Parser a = Parser + { unParser :: Ptr Word8 -- End position of the input + -> Ptr Word8 -- Current position in the input + -> IO (ParseResult a) + } + +data ParseResult a + = ParseSuccess + { _newPos :: !(Ptr Word8) -- ^ New position in the input + , unParserResult :: a + } + | ParseFailure String + +instance Functor ParseResult where + fmap f (ParseSuccess p x) = ParseSuccess p (f x) + fmap _ (ParseFailure s) = ParseFailure s + +instance Functor Parser where + fmap f (Parser g) = Parser $ \end cur -> fmap f <$> g end cur + +instance Applicative Parser where + pure x = Parser $ \_ cur -> return $ ParseSuccess cur x + (<*>) = ap + +instance Monad Parser where +#if !MIN_VERSION_base(4,13,0) + fail = Fail.fail +#endif + return = pure + Parser f >>= g = Parser $ \end pos -> f end pos >>= \case + ParseSuccess pos' x -> unParser (g x) end pos' + ParseFailure s -> return $ ParseFailure s + +instance Fail.MonadFail Parser where + fail s = Parser $ \_ _ -> return $ ParseFailure s diff --git a/src/Data/ProtoLens/Encoding/Parser/Unsafe.hs b/src/Data/ProtoLens/Encoding/Parser/Unsafe.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Encoding/Parser/Unsafe.hs @@ -0,0 +1,21 @@ +module Data.ProtoLens.Encoding.Parser.Unsafe + ( unsafeLiftIO ) where + +import Data.ProtoLens.Encoding.Parser.Internal + +-- | Runs an arbitrary @IO@ action inside a @Parser@. +-- The generated code uses this function to construct vectors +-- efficiently by incrementally building up mutable vectors. +-- +-- NOTE: This is unsafe since @runParser@ +-- is a pure function, which lets us lift arbitrary IO into +-- pure operations. +-- However, here are some guarantees that we do get: +-- +-- - For each individual call to 'runParser', the action +-- wrapped by 'unsafeLiftIO' will be called exactly once. +-- - Different calls to 'unsafeLiftIO' within the same call to +-- 'runParser' will be sequenced according to their order in the Parser +-- monad. +unsafeLiftIO :: IO a -> Parser a +unsafeLiftIO m = Parser $ \_ p -> ParseSuccess p <$> m diff --git a/src/Data/ProtoLens/Encoding/Wire.hs b/src/Data/ProtoLens/Encoding/Wire.hs --- a/src/Data/ProtoLens/Encoding/Wire.hs +++ b/src/Data/ProtoLens/Encoding/Wire.hs @@ -1,122 +1,145 @@ --- Copyright 2016 Google Inc. All Rights Reserved. +-- | Module defining the individual base wire types (e.g. VarInt, Fixed64). -- --- Use of this source code is governed by a BSD-style --- license that can be found in the LICENSE file or at --- https://developers.google.com/open-source/licenses/bsd - -{-# LANGUAGE GADTs #-} -{-# LANGUAGE RankNTypes #-} --- | Module defining the individual base wire types (e.g. VarInt, Fixed64) and --- how to encode/decode them. -module Data.ProtoLens.Encoding.Wire( - WireType(..), - SomeWireType(..), - WireValue(..), - TaggedValue(..), - getTaggedValue, - putTaggedValue, - getWireValue, - putWireValue, - Equal(..), - equalWireTypes, +-- They are used to represent the @unknownFields@ within the proto message. +-- +-- Upstream docs: +-- +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE CPP #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +module Data.ProtoLens.Encoding.Wire + ( Tag(..) + , TaggedValue(..) + , WireValue(..) + , FieldSet + , splitTypeAndTag + , joinTypeAndTag + , parseFieldSet + , buildFieldSet + , buildMessageSet + , parseTaggedValueFromWire + , parseMessageSetTaggedValueFromWire ) where -import Data.Attoparsec.ByteString as Parse -import Data.Bits +import Control.DeepSeq (NFData(..)) +import Data.Bits ((.&.), (.|.), shiftL, shiftR) import qualified Data.ByteString as B -import Data.ByteString.Lazy.Builder as Builder -import Data.Monoid ((<>)) -import Data.Word +#if !MIN_VERSION_base(4,11,0) +import Data.Semigroup ((<>)) +#endif +import Data.Word (Word8, Word32, Word64) import Data.ProtoLens.Encoding.Bytes -data WireType a where - VarInt :: WireType Word64 - Fixed64 :: WireType Word64 - Fixed32 :: WireType Word32 - Lengthy :: WireType B.ByteString - StartGroup :: WireType () - EndGroup :: WireType () +-- | A tag that identifies a particular field of the message when converting +-- to/from the wire format. +newtype Tag = Tag { unTag :: Int } + deriving (Show, Eq, Ord, Num, NFData) --- A value read from the wire -data WireValue = forall a . WireValue !(WireType a) !a --- The wire contents of a single key-value pair in a Message. -data TaggedValue = TaggedValue !Int !WireValue +-- | The encoding of some unknown field on the wire. +data WireValue + = VarInt !Word64 + | Fixed64 !Word64 + | Lengthy !B.ByteString + | StartGroup + | EndGroup + | Fixed32 !Word32 + deriving (Eq, Ord) -instance Show (WireType a) where - show = show . wireTypeToInt +-- | A pair of an encoded field and a value. +data TaggedValue = TaggedValue !Tag !WireValue + deriving (Eq, Ord) -data Equal a b where - Equal :: Equal a a +type FieldSet = [TaggedValue] --- Assert that two wire types are the same, or fail with a message about this --- field. -{-# INLINE equalWireTypes #-} -equalWireTypes :: Monad m => String -> WireType a -> WireType b - -> m (Equal a b) -equalWireTypes _ VarInt VarInt = return Equal -equalWireTypes _ Fixed64 Fixed64 = return Equal -equalWireTypes _ Fixed32 Fixed32 = return Equal -equalWireTypes _ Lengthy Lengthy = return Equal -equalWireTypes _ StartGroup StartGroup = return Equal -equalWireTypes _ EndGroup EndGroup = return Equal -equalWireTypes name expected actual - = fail $ "Field " ++ name ++ " expects wire type " ++ show expected - ++ " but found " ++ show actual +-- TaggedValue, WireValue and Tag are strict, so their NFData instances are +-- trivial: +instance NFData TaggedValue where + rnf = (`seq` ()) -getWireValue :: WireType a -> Int -> Parser a -getWireValue VarInt _ = getVarInt -getWireValue Fixed64 _ = anyBits -getWireValue Fixed32 _ = anyBits -getWireValue Lengthy _ = getVarInt >>= Parse.take . fromIntegral -getWireValue StartGroup _ = return () -getWireValue EndGroup _ = return () +instance NFData WireValue where + rnf = (`seq` ()) -putWireValue :: WireType a -> a -> Builder -putWireValue VarInt n = putVarInt n -putWireValue Fixed64 n = word64LE n -putWireValue Fixed32 n = word32LE n -putWireValue Lengthy b = putVarInt (fromIntegral $ B.length b) <> byteString b -putWireValue StartGroup _ = mempty -putWireValue EndGroup _ = mempty +buildTaggedValue :: TaggedValue -> Builder +buildTaggedValue (TaggedValue tag wv) = + putVarInt (joinTypeAndTag tag (wireValueToInt wv)) + <> buildWireValue wv -data SomeWireType where - SomeWireType :: WireType a -> SomeWireType +-- builds in legacy MessageSet format. +-- See https://github.com/protocolbuffers/protobuf/blob/dec4939439d9ca2adf2bb14edccf876c2587faf2/src/google/protobuf/descriptor.proto#L444 +buildTaggedValueAsMessageSet :: TaggedValue -> Builder +buildTaggedValueAsMessageSet (TaggedValue (Tag t) wv) = + buildTaggedValue ( TaggedValue 1 StartGroup) + <> buildTaggedValue (TaggedValue 2 (VarInt $ fromIntegral t)) + <> buildTaggedValue (TaggedValue 3 wv) + <> buildTaggedValue (TaggedValue 1 EndGroup) -wireTypeToInt :: WireType a -> Word64 -wireTypeToInt VarInt = 0 -wireTypeToInt Fixed64 = 1 -wireTypeToInt Lengthy = 2 -wireTypeToInt StartGroup = 3 -wireTypeToInt EndGroup = 4 -wireTypeToInt Fixed32 = 5 +buildWireValue :: WireValue -> Builder +buildWireValue (VarInt w) = putVarInt w +buildWireValue (Fixed64 w) = putFixed64 w +buildWireValue (Fixed32 w) = putFixed32 w +buildWireValue (Lengthy b) = + putVarInt (fromIntegral $ B.length b) + <> putBytes b +buildWireValue StartGroup = mempty +buildWireValue EndGroup = mempty -intToWireType :: Word64 -> Either String SomeWireType -intToWireType 0 = Right $ SomeWireType VarInt -intToWireType 1 = Right $ SomeWireType Fixed64 -intToWireType 2 = Right $ SomeWireType Lengthy -intToWireType 3 = Right $ SomeWireType StartGroup -intToWireType 4 = Right $ SomeWireType EndGroup -intToWireType 5 = Right $ SomeWireType Fixed32 -intToWireType n = Left $ "Unrecognized wire type " ++ show n +wireValueToInt :: WireValue -> Word8 +wireValueToInt VarInt{} = 0 +wireValueToInt Fixed64{} = 1 +wireValueToInt Lengthy{} = 2 +wireValueToInt StartGroup{} = 3 +wireValueToInt EndGroup{} = 4 +wireValueToInt Fixed32{} = 5 -putTypeAndTag :: WireType a -> Int -> Builder -putTypeAndTag wt tag - = putVarInt $ wireTypeToInt wt .|. fromIntegral tag `shiftL` 3 +parseTaggedValue :: Parser TaggedValue +parseTaggedValue = getVarInt >>= parseTaggedValueFromWire -getTypeAndTag :: Parser (SomeWireType, Int) -getTypeAndTag = do - n <- getVarInt - case intToWireType (n .&. 7) of - Left err -> fail err - Right wt -> return (wt, fromIntegral $ n `shiftR` 3) +parseTaggedValueFromWire :: Word64 -> Parser TaggedValue +parseTaggedValueFromWire t = + let (tag, w) = splitTypeAndTag t + in TaggedValue tag <$> case w of + 0 -> VarInt <$> getVarInt + 1 -> Fixed64 <$> getFixed64 + 2 -> Lengthy <$> do + len <- getVarInt + getBytes $ fromIntegral len + 3 -> return StartGroup + 4 -> return EndGroup + 5 -> Fixed32 <$> getFixed32 + _ -> fail $ "Unknown wire type " ++ show w -getTaggedValue :: Parser TaggedValue -getTaggedValue = do - (SomeWireType wt, tag) <- getTypeAndTag - val <- getWireValue wt tag - return $ TaggedValue tag (WireValue wt val) +parseMessageSetTaggedValueFromWire :: Word64 -> Parser TaggedValue +parseMessageSetTaggedValueFromWire t = + parseTaggedValueFromWire t >>= \v -> case v of + TaggedValue 1 StartGroup -> parseTaggedValue >>= \ft -> case ft of + TaggedValue 2 (VarInt f) -> parseTaggedValue >>= \dt -> case dt of + TaggedValue 3 (Lengthy b) -> parseTaggedValue >>= \et -> case et of + TaggedValue 1 EndGroup -> return $ TaggedValue (Tag $ fromIntegral f) (Lengthy b) + _ -> fail "missing end_group" + _ -> fail "missing message" + _ -> fail "missing field tag" + _ -> return v -putTaggedValue :: TaggedValue -> Builder -putTaggedValue (TaggedValue tag (WireValue wt val)) = - putTypeAndTag wt tag <> putWireValue wt val +splitTypeAndTag :: Word64 -> (Tag, Word8) +splitTypeAndTag w = (fromIntegral $ w `shiftR` 3, fromIntegral (w .&. 7)) + +joinTypeAndTag :: Tag -> Word8 -> Word64 +joinTypeAndTag (Tag t) w = fromIntegral t `shiftL` 3 .|. fromIntegral w + +parseFieldSet :: Parser FieldSet +parseFieldSet = loop [] + where + loop ws = do + end <- atEnd + if end + then return $! reverse ws + else do + !w <- parseTaggedValue + loop (w:ws) + +buildFieldSet :: FieldSet -> Builder +buildFieldSet = mconcat . map buildTaggedValue + +buildMessageSet :: FieldSet -> Builder +buildMessageSet = mconcat . map buildTaggedValueAsMessageSet diff --git a/src/Data/ProtoLens/Field.hs b/src/Data/ProtoLens/Field.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Field.hs @@ -0,0 +1,42 @@ +{- | An implementation of overloaded record fields. This module +enables different types in the same module to have fields of the +same name. + +To use instances from this class, either: + +* Enable the @OverloadedLabels@ extension and + @import Data.ProtoLens.Labels ()@; +* Use the 'field' function along with the @TypeApplications@ extension; or, +* Import the corresponding names from the autogenerated @*_Fields@ module. + +For more information, see . +-} +{-# LANGUAGE AllowAmbiguousTypes #-} +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE FunctionalDependencies #-} +{-# LANGUAGE KindSignatures #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE ScopedTypeVariables #-} +module Data.ProtoLens.Field + ( HasField(..) + , field + ) where + +import GHC.Prim (Proxy#, proxy#) +import GHC.TypeLits (Symbol) + +-- | A lens for a given field. For example: +-- +-- > view field@"abc" x +-- > set field@"abc" 42 x +field :: forall x s a f . (HasField s x a, Functor f) => (a -> f a) -> s -> f s +field = fieldOf (proxy# :: Proxy# x) + +-- | A type class for lens fields. +-- +-- The instance @HasField s x a@ can be understood as "@s@ has a field named @x@ +-- of type @a@". +class HasField s (x :: Symbol) a | s x -> a where + fieldOf :: Functor f => Proxy# x -> (a -> f a) -> s -> f s diff --git a/src/Data/ProtoLens/Labels.hs b/src/Data/ProtoLens/Labels.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Labels.hs @@ -0,0 +1,38 @@ +{-# LANGUAGE CPP #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MagicHash #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE TypeOperators #-} +{-# LANGUAGE UndecidableInstances #-} +#if __GLASGOW_HASKELL__ >= 802 +{-# LANGUAGE ScopedTypeVariables #-} +#endif +{-# OPTIONS_GHC -fno-warn-orphans #-} + +------------------------------------------------------------------------------ +-- | This module provides OverloadedLabels 'IsLabel' support via an +-- orphan instance. This means a @Lens.Family.Lens@ can be referenced +-- as @#foo@ whenever we have an instance of 'Data.ProtoLens.Field.HasField' +-- with the label @"foo"@." +-- +-- To use these labels, enable the @OverloadedLabels@ extension, and then +-- import: +-- +-- > import Data.ProtoLens.Labels() +module Data.ProtoLens.Labels where + +import GHC.OverloadedLabels (IsLabel (..)) +#if __GLASGOW_HASKELL__ >= 802 +import GHC.Prim (Proxy#, proxy#) +#endif + +import Data.ProtoLens.Field (HasField(..)) + +instance (HasField s x a, p ~ (a -> f a), q ~ (s -> f s), Functor f, + a ~ b) => IsLabel x (p -> q) where +#if __GLASGOW_HASKELL__ >= 802 + fromLabel = fieldOf (proxy# :: Proxy# x) +#else + fromLabel x = fieldOf x +#endif diff --git a/src/Data/ProtoLens/Message.hs b/src/Data/ProtoLens/Message.hs --- a/src/Data/ProtoLens/Message.hs +++ b/src/Data/ProtoLens/Message.hs @@ -6,14 +6,18 @@ {-# LANGUAGE GADTs #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PatternGuards #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE TypeApplications #-} -- | Datatypes for reflection of protocol buffer messages. module Data.ProtoLens.Message ( -- * Reflection of Messages Message(..), Tag(..), - MessageDescriptor(..), + allFields, FieldDescriptor(..), fieldDescriptorName, isRequired, @@ -21,55 +25,106 @@ WireDefault(..), Packing(..), FieldTypeDescriptor(..), + ScalarField(..), + MessageOrGroup(..), FieldDefault(..), MessageEnum(..), - -- * Building protocol buffers - Default(..), + -- * Constructing protocol buffers build, + -- * Proto registries + Registry, + register, + lookupRegistered, + SomeMessageType(..), + -- * Any messages + matchAnyMessage, + AnyMessageDescriptor(..), -- * Utilities for constructing protocol buffer lenses maybeLens, -- * Internal utilities for parsing protocol buffers reverseRepeatedFields, + -- * Unknown fields + FieldSet, + TaggedValue(..), + discardUnknownFields, ) where import qualified Data.ByteString as B -import Data.Default.Class import Data.Int import qualified Data.Map as Map import Data.Map (Map) import Data.Maybe (fromMaybe) +import Data.Proxy (Proxy(..)) import qualified Data.Text as T import Data.Word -import Lens.Family2 (Lens', over) +import Lens.Family2 (Lens', over, set) import Lens.Family2.Unchecked (lens) +import qualified Data.Semigroup as Semigroup +import Data.ProtoLens.Encoding.Bytes (Builder, Parser) +import Data.ProtoLens.Encoding.Wire + ( Tag(..) + , TaggedValue(..) + ) + -- | Every protocol buffer is an instance of 'Message'. This class enables -- serialization by providing reflection of all of the fields that may be used -- by this type. -class Default msg => Message msg where - descriptor :: MessageDescriptor msg +class Message msg where + -- | A unique identifier for this type, of the format + -- @"packagename.messagename"@. + messageName :: Proxy msg -> T.Text --- | The description of a particular protocol buffer message type. -data MessageDescriptor msg = MessageDescriptor - { messageName :: T.Text - -- ^ A unique identifier for this type, of the format - -- @"packagename.messagename"@. - , fieldsByTag :: Map Tag (FieldDescriptor msg) - -- ^ The fields of the proto, indexed by their (integer) tag. - , fieldsByTextFormatName :: Map String (FieldDescriptor msg) - -- ^ This map is keyed by the name of the field used for text format protos. - -- This is just the field name for every field except for group fields, - -- which use their Message type name in text protos instead of their - -- field name. For example, "optional group Foo" has the field name "foo" - -- but in this map it is stored with the key "Foo". - } + -- | The serialized protobuffer message descriptor for this type. + -- + -- For a friendlier version which returns the actual descriptor type, + -- use @Data.ProtoLens.Descriptor.messageDescriptor@ + -- from the @proto-lens-protobuf-types@ package. + packedMessageDescriptor :: Proxy msg -> B.ByteString --- | A tag that identifies a particular field of the message when converting --- to/from the wire format. -newtype Tag = Tag { unTag :: Int} - deriving (Show, Eq, Ord, Num) + -- | The serialized protobuffer file message descriptor containing this type. + -- + -- For a friendlier version which returns the actual file descriptor type, + -- use @Data.ProtoLens.Descriptor.fileDescriptor@ + -- from the @proto-lens-protobuf-types@ package. + packedFileDescriptor :: Proxy msg -> B.ByteString + -- | A message with all fields set to their default values. + -- + -- Satisfies @encodeMessage defMessage == ""@ and @decodeMessage "" == Right defMessage@. + defMessage :: msg + -- | The fields of the proto, indexed by their (integer) tag. + fieldsByTag :: Map Tag (FieldDescriptor msg) + + -- | This map is keyed by the name of the field used for text format protos. + -- This is just the field name for every field except for group fields, + -- which use their Message type name in text protos instead of their + -- field name. For example, "optional group Foo" has the field name "foo" + -- but in this map it is stored with the key "Foo". + fieldsByTextFormatName :: Map String (FieldDescriptor msg) + fieldsByTextFormatName = + Map.fromList [(n, f) | f@(FieldDescriptor n _ _) <- allFields] + + -- | Access the unknown fields of a Message. + unknownFields :: Lens' msg FieldSet + + -- | Decode a message value. + -- + -- See also the functions in "Data.ProtoLens.Encoding". + parseMessage :: Parser msg + + -- | Encode a message value. + -- + -- See also the functions in "Data.ProtoLens.Encoding". + buildMessage :: msg -> Builder + +allFields :: Message msg => [FieldDescriptor msg] +allFields = Map.elems fieldsByTag + +-- TODO: represent FieldSet as a Vector too. +type FieldSet = [TaggedValue] + -- | A description of a specific field of a protocol buffer. -- -- The 'String' parameter is the name of the field from the .proto file, @@ -119,8 +174,9 @@ -- | A proto3 field type with an implicit default value. -- --- This is distinct from 'Data.Default' to avoid orphan instances, and because --- 'Bool' doesn't necessarily have a good Default instance for general usage. +-- This is distinct from, say, 'Data.Default' to avoid orphan instances, and +-- because 'Bool' doesn't necessarily have a good Default instance for general +-- usage. class FieldDefault value where fieldDefault :: value @@ -151,39 +207,62 @@ instance FieldDefault T.Text where fieldDefault = T.empty - -- | How a given repeated field is transmitted on the wire format. data Packing = Packed | Unpacked -- | A description of the type of a given field value. data FieldTypeDescriptor value where - MessageField :: Message value => FieldTypeDescriptor value - GroupField :: Message value => FieldTypeDescriptor value - EnumField :: MessageEnum value => FieldTypeDescriptor value - Int32Field :: FieldTypeDescriptor Int32 - Int64Field :: FieldTypeDescriptor Int64 - UInt32Field :: FieldTypeDescriptor Word32 - UInt64Field :: FieldTypeDescriptor Word64 - SInt32Field :: FieldTypeDescriptor Int32 - SInt64Field :: FieldTypeDescriptor Int64 - Fixed32Field :: FieldTypeDescriptor Word32 - Fixed64Field :: FieldTypeDescriptor Word64 - SFixed32Field :: FieldTypeDescriptor Int32 - SFixed64Field :: FieldTypeDescriptor Int64 - FloatField :: FieldTypeDescriptor Float - DoubleField :: FieldTypeDescriptor Double - BoolField :: FieldTypeDescriptor Bool - StringField :: FieldTypeDescriptor T.Text - BytesField :: FieldTypeDescriptor B.ByteString + MessageField :: Message value => MessageOrGroup -> FieldTypeDescriptor value + ScalarField :: ScalarField value -> FieldTypeDescriptor value deriving instance Show (FieldTypeDescriptor value) +data MessageOrGroup = MessageType | GroupType + deriving Show + +data ScalarField t where + EnumField :: MessageEnum value => ScalarField value + Int32Field :: ScalarField Int32 + Int64Field :: ScalarField Int64 + UInt32Field :: ScalarField Word32 + UInt64Field :: ScalarField Word64 + SInt32Field :: ScalarField Int32 + SInt64Field :: ScalarField Int64 + Fixed32Field :: ScalarField Word32 + Fixed64Field :: ScalarField Word64 + SFixed32Field :: ScalarField Int32 + SFixed64Field :: ScalarField Int64 + FloatField :: ScalarField Float + DoubleField :: ScalarField Double + BoolField :: ScalarField Bool + StringField :: ScalarField T.Text + BytesField :: ScalarField B.ByteString + +deriving instance Show (ScalarField value) + +matchAnyMessage :: forall value . FieldTypeDescriptor value -> Maybe (AnyMessageDescriptor value) +matchAnyMessage (MessageField _) + | messageName (Proxy @value) == "google.protobuf.Any" + , Just (FieldDescriptor _ (ScalarField StringField) (PlainField Optional typeUrlLens)) + <- Map.lookup 1 (fieldsByTag @value) + , Just (FieldDescriptor _ (ScalarField BytesField) (PlainField Optional valueLens)) + <- Map.lookup 2 (fieldsByTag @value) + = Just $ AnyMessageDescriptor typeUrlLens valueLens +matchAnyMessage _ = Nothing + +data AnyMessageDescriptor msg + = AnyMessageDescriptor + { anyTypeUrlLens :: Lens' msg T.Text + , anyValueLens :: Lens' msg B.ByteString + } + -- | A class for protocol buffer enums that enables safe decoding. class (Enum a, Bounded a) => MessageEnum a where -- | Convert the given 'Int' to an enum value. Returns 'Nothing' if -- no corresponding value was defined in the .proto file. maybeToEnum :: Int -> Maybe a - -- | Get the name of this enum as defined in the .proto file. + -- | Get the name of this enum as defined in the .proto file. Used + -- for the human-readable output in @Data.ProtoLens.TextFormat@. showEnum :: a -> String -- | Convert the given 'String' to an enum value. Returns 'Nothing' if -- no corresponding value was defined in the .proto file. @@ -196,8 +275,8 @@ -- > x, y :: Lens' A Int -- > m :: A -- > m = build ((x .~ 5) . (y .~ 7)) -build :: Default a => (a -> a) -> a -build = ($ def) +build :: Message a => (a -> a) -> a +build = ($ defMessage) -- | A helper lens for accessing optional fields. -- This is used as part of code generation, and should generally not be needed @@ -231,3 +310,32 @@ reverseListField x (FieldDescriptor _ _ (RepeatedField _ f)) = over f reverse x reverseListField x _ = x + +-- | A set of known message types. Can help encode/decode protobufs containing +-- @Data.ProtoLens.Any@ values in a more human-readable text format. +-- +-- Registries can be combined using their 'Monoid' instance. +-- +-- See the @withRegistry@ functions in 'Data.ProtoLens.TextFormat' +newtype Registry = Registry (Map.Map T.Text SomeMessageType) + deriving (Semigroup.Semigroup, Monoid) + +-- | Build a 'Registry' containing a single proto type. +-- +-- Example: +-- > register (Proxy :: Proxy Proto.My.Proto.Type) +register :: forall msg . Message msg => Proxy msg -> Registry +register p = Registry $ Map.singleton (messageName (Proxy @msg)) (SomeMessageType p) + +-- | Look up a message type by name (e.g., +-- @"type.googleapis.com/google.protobuf.FloatValue"@). The URL corresponds to +-- the field @google.protobuf.Any.type_url@. +lookupRegistered :: T.Text -> Registry -> Maybe SomeMessageType +lookupRegistered n (Registry m) = Map.lookup (snd $ T.breakOnEnd "/" n) m + +data SomeMessageType where + SomeMessageType :: Message msg => Proxy msg -> SomeMessageType + +-- TODO: recursively +discardUnknownFields :: Message msg => msg -> msg +discardUnknownFields = set unknownFields [] diff --git a/src/Data/ProtoLens/Prism.hs b/src/Data/ProtoLens/Prism.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Prism.hs @@ -0,0 +1,80 @@ +-- Copyright 2016 Google Inc. All Rights Reserved. +-- +-- Use of this source code is governed by a BSD-style +-- license that can be found in the LICENSE file or at +-- https://developers.google.com/open-source/licenses/bsd + +-- | This module defines the 'Prism' type and combinators. Used for building +-- 'Prism's for oneof fields. +{-# LANGUAGE RankNTypes #-} + +module Data.ProtoLens.Prism + ( Prism + , Prism' + , AReview + , (#) + , prism + , prism' + , _Left + , _Right + , _Just + , _Nothing + ) where + +import Data.Tagged (Tagged (..)) +import Data.Functor.Identity (Identity (..)) +import Data.Profunctor (dimap) +import Data.Profunctor.Choice +import Data.Profunctor.Unsafe ((#.), (.#)) + + +------------------------------------------------------------------------------ +-- Prism Internals +------------------------------------------------------------------------------ +type Prism s t a b = forall p f. (Choice p, Applicative f) => p a (f b) -> p s (f t) + +type Prism' s a = Prism s s a a + +type Optic p f s t a b = p a (f b) -> p s (f t) + +type Optic' p f s a = Optic p f s s a a + +type AReview t b = Optic' Tagged Identity t b + +-- | Used for constructing 'Prism' values. +-- +-- @'_Just' '#' 5 == 'Just' 5@ +( # ) :: AReview t b -> b -> t +( # ) p = runIdentity #. unTagged #. p .# Tagged .# Identity +infixr 8 # + + +------------------------------------------------------------------------------ +-- Prism Combinators +------------------------------------------------------------------------------ + +-- | Build a 'Control.Lens.Prism.Prism'. +-- +-- @'Either' t a@ is used instead of @'Maybe' a@ to permit the types of @s@ and @t@ to differ. +-- +prism :: (b -> t) -> (s -> Either t a) -> Prism s t a b +prism bt seta = dimap seta (either pure (fmap bt)) . right' +{-# INLINE prism #-} + +-- | This is usually used to build a 'Prism'', when you have to use an operation like +-- 'Data.Typeable.cast' which already returns a 'Maybe'. +prism' :: (b -> s) -> (s -> Maybe a) -> Prism s s a b +prism' bs sma = prism bs (\s -> maybe (Left s) Right (sma s)) +{-# INLINE prism' #-} + +_Left :: Prism (Either a c) (Either b c) a b +_Left = prism Left $ either Right (Left . Right) + +_Right :: Prism (Either c a) (Either c b) a b +_Right = prism Right $ either (Left . Left) Right + +_Just :: Prism (Maybe a) (Maybe b) a b +_Just = prism Just $ maybe (Left Nothing) Right + +_Nothing :: Prism' (Maybe a) () +_Nothing = prism' (const Nothing) $ maybe (Just ()) (const Nothing) diff --git a/src/Data/ProtoLens/Service/Types.hs b/src/Data/ProtoLens/Service/Types.hs new file mode 100644 --- /dev/null +++ b/src/Data/ProtoLens/Service/Types.hs @@ -0,0 +1,128 @@ +{-# LANGUAGE ConstraintKinds #-} +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE MultiParamTypeClasses #-} +{-# LANGUAGE PolyKinds #-} +{-# LANGUAGE RankNTypes #-} +{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE TypeOperators #-} +{-# LANGUAGE UndecidableInstances #-} +{-# OPTIONS_GHC -fno-warn-redundant-constraints #-} + +-- | This module provides typeclasses for describing protobuf service metadata. +-- It is intended to be used by library authors to generating bindings against +-- proto services for specific RPC backends. +module Data.ProtoLens.Service.Types + ( Service (..) + , HasAllMethods + , HasMethodImpl (..) + , HasMethod + , StreamingType (..) + ) where + +import qualified Data.ByteString as B +import Data.Kind (Constraint, Type) +import Data.ProtoLens.Message (Message) +import Data.Proxy (Proxy(..)) +import GHC.TypeLits + + +-- | Reifies the fact that there is a 'HasMethod' instance for every symbol +-- claimed by the 'ServiceMethods' associated type. +class HasAllMethods s (ms :: [Symbol]) +instance HasAllMethods s '[] +instance (HasAllMethods s ms, HasMethodImpl s m) => HasAllMethods s (m ': ms) + + +-- | Metadata describing a protobuf service. The 's' parameter is a phantom +-- type generated by proto-lens. +-- +-- The 'ServiceName' and 'ServicePackage' associated type can be used to +-- generate RPC endpoint paths. +-- +-- 'ServiceMethods' is a promoted list containing every method defined on the +-- service. As witnessed by the 'HasAllMethods' superclass constraint here, +-- this type can be used to discover every instance of 'HasMethod' available +-- for the service. +class ( KnownSymbol (ServiceName s) + , KnownSymbol (ServicePackage s) + , HasAllMethods s (ServiceMethods s) + ) => Service s where + type ServiceName s :: Symbol + type ServicePackage s :: Symbol + type ServiceMethods s :: [Symbol] + + packedServiceDescriptor :: Proxy s -> B.ByteString + +------------------------------------------------------------------------------ +-- | Data type to be used as a promoted type for 'MethodStreamingType'. +data StreamingType + = NonStreaming + | ClientStreaming + | ServerStreaming + | BiDiStreaming + deriving (Eq, Ord, Enum, Bounded, Read, Show) + + +-- | Metadata describing a service method. The 'MethodInput' and 'MethodOutput' +-- type families correspond to the 'Message's generated by proto-lens for the +-- RPC as described in the protobuf. +-- +-- 'IsClientStreaming' and 'IsServerStreaming' can be used to dispatch on +-- library code which wishes to provide different interfaces depending on the +-- type of streaming of the method. +-- +-- Library code should use 'HasMethod' instead of this class directly whenever +-- the constraint will be exposed to the end user. 'HasMethod' provides +-- substantially friendlier error messages when used incorrectly. +class ( KnownSymbol m + , KnownSymbol (MethodName s m) + , Service s + , Message (MethodInput s m) + , Message (MethodOutput s m) + ) => HasMethodImpl s (m :: Symbol) where + type MethodName s m :: Symbol + type MethodInput s m :: Type + type MethodOutput s m :: Type + type MethodStreamingType s m :: StreamingType + + +-- | Helper constraint that expands to a user-friendly error message when 'm' +-- isn't actually a method provided by service 's'. +type HasMethod s m = + ( RequireHasMethod s m (ListContains m (ServiceMethods s)) + , HasMethodImpl s m + ) + + +-- | Outputs an error message saying that the given method wasn't found, and +-- suggests alternatives the user might have wanted. +type family RequireHasMethod s (m :: Symbol) (h :: Bool) :: Constraint where + RequireHasMethod s m 'False = TypeError + ( 'Text "No method " + ':<>: 'ShowType m + ':<>: 'Text " available for service '" + ':<>: 'ShowType s + ':<>: 'Text "'." + ':$$: 'Text "Available methods are: " + ':<>: ShowList (ServiceMethods s) + ) + RequireHasMethod s m 'True = () + + +-- | Expands to 'True' when 'n' is in promoted list 'hs', 'False' otherwise. +type family ListContains (n :: k) (hs :: [k]) :: Bool where + ListContains n '[] = 'False + ListContains n (n ': hs) = 'True + ListContains n (x ': hs) = ListContains n hs + + +-- | Pretty prints a promoted list. +type family ShowList (ls :: [k]) :: ErrorMessage where + ShowList '[] = 'Text "" + ShowList '[x] = 'ShowType x + ShowList (x ': xs) = + 'ShowType x ':<>: 'Text ", " ':<>: ShowList xs + diff --git a/src/Data/ProtoLens/TextFormat.hs b/src/Data/ProtoLens/TextFormat.hs --- a/src/Data/ProtoLens/TextFormat.hs +++ b/src/Data/ProtoLens/TextFormat.hs @@ -5,33 +5,50 @@ -- https://developers.google.com/open-source/licenses/bsd -- | Functions for converting protocol buffers to a human-readable text format. +{-# LANGUAGE CPP #-} {-# LANGUAGE GADTs #-} +{-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE PatternGuards #-} {-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE TypeApplications #-} module Data.ProtoLens.TextFormat( showMessage, + showMessageWithRegistry, showMessageShort, pprintMessage, + pprintMessageWithRegistry, readMessage, + readMessageWithRegistry, readMessageOrDie, ) where -import Lens.Family2 ((&),(^.),(.~), set, over) +import Lens.Family2 ((&),(^.),(.~), set, over, view) import Control.Arrow (left) +import Data.Bifunctor (first) import qualified Data.ByteString import Data.Char (isPrint, isAscii, chr) -import Data.Foldable (foldlM, foldl') -import Data.Maybe (catMaybes) +import Data.Foldable (foldlM) +import qualified Data.Foldable as F import qualified Data.Map as Map +import Data.Maybe (catMaybes) +import Data.Proxy (Proxy(Proxy)) import qualified Data.Set as Set import qualified Data.Text.Encoding as Text import qualified Data.Text.Lazy as Lazy +import qualified Data.Text as Text (unpack) import Numeric (showOct) import Text.Parsec (parse) import Text.PrettyPrint -import Data.ProtoLens.Message +#if MIN_VERSION_base(4,11,0) +import Prelude hiding ((<>)) +#endif + +import Data.ProtoLens.Encoding (decodeMessage, encodeMessage) +import Data.ProtoLens.Encoding.Bytes (runParser) +import Data.ProtoLens.Encoding.Wire +import Data.ProtoLens.Message hiding (buildMessage, parseMessage) import qualified Data.ProtoLens.TextFormat.Parser as Parser -- TODO: This code is newer and missing some edge cases, @@ -52,28 +69,36 @@ -- | Pretty-print the given message into a human-readable form. pprintMessage :: Message msg => msg -> Doc -pprintMessage = pprintMessage' descriptor +pprintMessage = pprintMessageWithRegistry mempty +-- | Pretty-print the given message into human-readable form, using the given +-- 'Registry' to decode @google.protobuf.Any@ values. +pprintMessageWithRegistry :: Message msg => Registry -> msg -> Doc +pprintMessageWithRegistry reg msg + -- Either put all fields together on a single line, or use a separate line + -- for each field. We use a single "sep" for all fields (and all elements + -- of all the repeated fields) to avoid putting some repeated fields on one + -- line and other fields on multiple lines, which is less readable. + = sep $ concatMap (pprintField reg msg) allFields + ++ map pprintTaggedValue (msg ^. unknownFields) + -- | Convert the given message into a human-readable 'String'. showMessage :: Message msg => msg -> String showMessage = render . pprintMessage +-- | Convert the given message into a human-readable 'String', using the +-- 'Registry' to encode @google.protobuf.Any@ values. +showMessageWithRegistry :: Message msg => Registry -> msg -> String +showMessageWithRegistry reg = render . pprintMessageWithRegistry reg + -- | Serializes a proto as a string on a single line. Useful for debugging -- and error messages like @.DebugString()@ in other languages. showMessageShort :: Message msg => msg -> String showMessageShort = renderStyle (Style OneLineMode maxBound 1.5) . pprintMessage -pprintMessage' :: MessageDescriptor msg -> msg -> Doc -pprintMessage' descr msg - -- Either put all fields together on a single line, or use a separate line - -- for each field. We use a single "sep" for all fields (and all elements - -- of all the repeated fields) to avoid putting some repeated fields on one - -- line and other fields on multiple lines, which is less readable. - = sep $ concatMap (pprintField msg) $ Map.elems $ fieldsByTag descr - -pprintField :: msg -> FieldDescriptor msg -> [Doc] -pprintField msg (FieldDescriptor name typeDescr accessor) - = map (pprintFieldValue name typeDescr) $ case accessor of +pprintField :: Registry -> msg -> FieldDescriptor msg -> [Doc] +pprintField reg msg (FieldDescriptor name typeDescr accessor) + = map (pprintFieldValue reg name typeDescr) $ case accessor of PlainField d f | Optional <- d, val == fieldDefault -> [] | otherwise -> [val] @@ -82,31 +107,54 @@ -- TODO: better printing for packed fields RepeatedField _ f -> msg ^. f MapField k v f -> pairToMsg <$> Map.assocs (msg ^. f) - where pairToMsg (x,y) = def & k .~ x - & v .~ y + where pairToMsg (x,y) = defMessage + & k .~ x + & v .~ y -pprintFieldValue :: String -> FieldTypeDescriptor value -> value -> Doc -pprintFieldValue name MessageField m - = sep [text name <+> lbrace, nest 2 (pprintMessage m), rbrace] -pprintFieldValue name EnumField x = text name <> colon <+> text (showEnum x) -pprintFieldValue name Int32Field x = primField name x -pprintFieldValue name Int64Field x = primField name x -pprintFieldValue name UInt32Field x = primField name x -pprintFieldValue name UInt64Field x = primField name x -pprintFieldValue name SInt32Field x = primField name x -pprintFieldValue name SInt64Field x = primField name x -pprintFieldValue name Fixed32Field x = primField name x -pprintFieldValue name Fixed64Field x = primField name x -pprintFieldValue name SFixed32Field x = primField name x -pprintFieldValue name SFixed64Field x = primField name x -pprintFieldValue name FloatField x = primField name x -pprintFieldValue name DoubleField x = primField name x -pprintFieldValue name BoolField x = text name <> colon <+> boolValue x -pprintFieldValue name StringField x = pprintByteString name (Text.encodeUtf8 x) -pprintFieldValue name BytesField x = pprintByteString name x -pprintFieldValue name GroupField m - = text name <+> lbrace $$ nest 2 (pprintMessage m) $$ rbrace +pprintFieldValue :: Registry -> String -> FieldTypeDescriptor value -> value -> Doc +pprintFieldValue reg name field@(MessageField MessageType) m + | Just AnyMessageDescriptor { anyTypeUrlLens, anyValueLens } <- matchAnyMessage field, + typeUri <- view anyTypeUrlLens m, + fieldData <- view anyValueLens m, + Just (SomeMessageType (Proxy :: Proxy value')) <- lookupRegistered typeUri reg, + Right (anyValue :: value') <- decodeMessage fieldData = + pprintSubmessage name + $ sep + [ lbrack <> text (Text.unpack typeUri) <> rbrack <+> lbrace + , nest 2 (pprintMessageWithRegistry reg anyValue) + , rbrace ] + | otherwise = + pprintSubmessage name (pprintMessageWithRegistry reg m) +pprintFieldValue reg name (MessageField GroupType) m + = pprintSubmessage name (pprintMessageWithRegistry reg m) +pprintFieldValue _ name (ScalarField f) x = named name $ pprintScalarValue f x +named :: String -> Doc -> Doc +named n x = text n <> colon <+> x + + +pprintScalarValue :: ScalarField value -> value -> Doc +pprintScalarValue EnumField x = text (showEnum x) +pprintScalarValue Int32Field x = primField x +pprintScalarValue Int64Field x = primField x +pprintScalarValue UInt32Field x = primField x +pprintScalarValue UInt64Field x = primField x +pprintScalarValue SInt32Field x = primField x +pprintScalarValue SInt64Field x = primField x +pprintScalarValue Fixed32Field x = primField x +pprintScalarValue Fixed64Field x = primField x +pprintScalarValue SFixed32Field x = primField x +pprintScalarValue SFixed64Field x = primField x +pprintScalarValue FloatField x = primField x +pprintScalarValue DoubleField x = primField x +pprintScalarValue BoolField x = boolValue x +pprintScalarValue StringField x = pprintByteString (Text.encodeUtf8 x) +pprintScalarValue BytesField x = pprintByteString x + +pprintSubmessage :: String -> Doc -> Doc +pprintSubmessage name contents = + sep [text name <+> lbrace, nest 2 contents, rbrace] + -- | Formats a string in a way that mostly matches the C-compatible escaping -- used by the Protocol Buffer distribution. We depart a bit by escaping all -- non-ASCII characters, which depending on the locale, the distribution might @@ -116,8 +164,8 @@ -- and \\ only. Note that Haskell string-literal syntax calls for "\011" to be -- interpreted as decimal 11, rather than the decimal 9 it actually represent, -- so you can't use Prelude.read to parse the strings created here. -pprintByteString :: String -> Data.ByteString.ByteString -> Doc -pprintByteString name x = text name <> colon <+> char '\"' +pprintByteString :: Data.ByteString.ByteString -> Doc +pprintByteString x = char '\"' <> text (concatMap escape $ Data.ByteString.unpack x) <> char '\"' where escape w8 | ch == '\n' = "\\n" | ch == '\r' = "\\r" @@ -131,19 +179,35 @@ ch = chr $ fromIntegral w8 pad str = replicate (3 - length str) '0' ++ str -primField :: Show value => String -> value -> Doc -primField name x = text name <> colon <+> text (show x) +primField :: Show value => value -> Doc +primField x = text (show x) boolValue :: Bool -> Doc boolValue True = text "true" boolValue False = text "false" +pprintTaggedValue :: TaggedValue -> Doc +pprintTaggedValue (TaggedValue t wv) = case wv of + VarInt x -> named name $ primField x + Fixed64 x -> named name $ primField x + Fixed32 x -> named name $ primField x + Lengthy x -> case runParser parseFieldSet x of + Left _ -> named name $ pprintByteString x + Right ts -> pprintSubmessage name + $ sep $ map pprintTaggedValue ts + -- TODO: implement better printing for unknown groups + StartGroup -> named name $ text "start_group" + EndGroup -> named name $ text "end_group" + where + name = show (unTag t) + + -------------------------------------------------------------------------------- -- Parsing -- | Parse a 'Message' from the human-readable protocol buffer text format. readMessage :: Message msg => Lazy.Text -> Either String msg -readMessage str = left show (parse Parser.parser "" str) >>= buildMessage +readMessage = readMessageWithRegistry mempty -- | Parse a 'Message' from the human-readable protocol buffer text format. -- Throws an error if the parse was not successful. @@ -152,47 +216,49 @@ Left e -> error $ "readMessageOrDie: " ++ e Right x -> x -buildMessage :: forall msg . Message msg => Parser.Message -> Either String msg -buildMessage fields - | missing <- missingFields desc fields, not $ null missing +-- | Parse a 'Message' from a human-readable protocol buffer text format, using +-- the given 'Registry' to decode 'Any' fields +readMessageWithRegistry :: Message msg => Registry -> Lazy.Text -> Either String msg +readMessageWithRegistry reg str = left show (parse Parser.parser "" str) >>= buildMessage reg + +buildMessage :: forall msg . Message msg => Registry -> Parser.Message -> Either String msg +buildMessage reg fields + | missing <- missingFields (Proxy @msg) fields, not $ null missing = Left $ "Missing fields " ++ show missing - | otherwise = reverseRepeatedFields (fieldsByTag desc) - <$> buildMessageFromDescriptor desc def fields - where - desc :: MessageDescriptor msg - desc = descriptor + | otherwise = reverseRepeatedFields fieldsByTag + <$> buildMessageFromDescriptor reg defMessage fields -missingFields :: MessageDescriptor msg -> Parser.Message -> [String] -missingFields desc = Set.toList . foldl' deleteField requiredFieldNames +missingFields :: forall msg . Message msg => Proxy msg -> Parser.Message -> [String] +missingFields _ = Set.toList . F.foldl' deleteField requiredFieldNames where requiredFieldNames :: Set.Set String requiredFieldNames = Set.fromList $ Map.keys $ Map.filter isRequired - $ fieldsByTextFormatName desc + $ fieldsByTextFormatName @msg deleteField :: Set.Set String -> Parser.Field -> Set.Set String deleteField fs (Parser.Field (Parser.Key name) _) = Set.delete name fs deleteField fs (Parser.Field (Parser.UnknownKey n) _) - | Just d <- Map.lookup (Tag (fromIntegral n)) $ fieldsByTag desc + | Just d <- Map.lookup (Tag (fromIntegral n)) (fieldsByTag @msg) = Set.delete (fieldDescriptorName d) fs deleteField fs _ = fs buildMessageFromDescriptor - :: MessageDescriptor msg -> msg -> Parser.Message -> Either String msg -buildMessageFromDescriptor descr = foldlM (addField descr) + :: Message msg => Registry -> msg -> Parser.Message -> Either String msg +buildMessageFromDescriptor reg = foldlM (addField reg) -addField :: MessageDescriptor msg -> msg -> Parser.Field -> Either String msg -addField descr msg (Parser.Field key rawValue) = do - FieldDescriptor _ typeDescriptor accessor <- getFieldDescriptor - value <- makeValue typeDescriptor rawValue +addField :: forall msg . Message msg => Registry -> msg -> Parser.Field -> Either String msg +addField reg msg (Parser.Field key rawValue) = do + FieldDescriptor name typeDescriptor accessor <- getFieldDescriptor + value <- makeValue name reg typeDescriptor rawValue return $ modifyField accessor value msg where getFieldDescriptor | Parser.Key name <- key, Just f <- Map.lookup name - (fieldsByTextFormatName descr) + fieldsByTextFormatName = return f | Parser.UnknownKey tag <- key, Just f <- Map.lookup (fromIntegral tag) - (fieldsByTag descr) + fieldsByTag = return f | otherwise = Left $ "Unrecognized field " ++ show key @@ -203,37 +269,64 @@ modifyField (MapField key value f) mapElem = over f (Map.insert (mapElem ^. key) (mapElem ^. value)) -makeValue :: FieldTypeDescriptor value -> Parser.Value -> Either String value -makeValue Int32Field (Parser.IntValue x) = Right (fromInteger x) -makeValue Int64Field (Parser.IntValue x) = Right (fromInteger x) -makeValue UInt32Field (Parser.IntValue x) = Right (fromInteger x) -makeValue UInt64Field (Parser.IntValue x) = Right (fromInteger x) -makeValue SInt32Field (Parser.IntValue x) = Right (fromInteger x) -makeValue SInt64Field (Parser.IntValue x) = Right (fromInteger x) -makeValue Fixed32Field (Parser.IntValue x) = Right (fromInteger x) -makeValue Fixed64Field (Parser.IntValue x) = Right (fromInteger x) -makeValue SFixed32Field (Parser.IntValue x) = Right (fromInteger x) -makeValue SFixed64Field (Parser.IntValue x) = Right (fromInteger x) -makeValue FloatField (Parser.IntValue x) = Right (fromInteger x) -makeValue DoubleField (Parser.IntValue x) = Right (fromInteger x) -makeValue BoolField (Parser.IntValue x) +makeValue + :: forall value + . String -- ^ name of field + -> Registry + -> FieldTypeDescriptor value + -> Parser.Value + -> Either String value +makeValue name _ (ScalarField f) v = + first (("Error parsing field " ++ show name ++ ": ") ++) $ makeScalarValue f v +makeValue name reg field@(MessageField MessageType) (Parser.MessageValue (Just typeUri) x) + | Just AnyMessageDescriptor { anyTypeUrlLens, anyValueLens } <- matchAnyMessage field = + case lookupRegistered typeUri reg of + Nothing -> Left $ "Could not decode google.protobuf.Any for field " + ++ show name ++ ": unregistered type URI " + ++ show typeUri + Just (SomeMessageType (Proxy :: Proxy value')) -> + case buildMessage reg x :: Either String value' of + Left err -> Left err + Right value' -> Right (defMessage + & anyTypeUrlLens .~ typeUri + & anyValueLens .~ encodeMessage value') + | otherwise = Left ("Type mismatch parsing explicitly typed message. Expected " ++ + show (messageName (Proxy @value)) ++ + ", got " ++ show typeUri) +makeValue _ reg (MessageField _) (Parser.MessageValue _ x) = buildMessage reg x +makeValue name _ (MessageField _) val = + Left $ "Type mismatch for field " ++ show name ++ + ": expected message, found " ++ show val + +makeScalarValue :: ScalarField value -> Parser.Value -> Either String value +makeScalarValue Int32Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue Int64Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue UInt32Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue UInt64Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue SInt32Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue SInt64Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue Fixed32Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue Fixed64Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue SFixed32Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue SFixed64Field (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue FloatField (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue DoubleField (Parser.IntValue x) = Right (fromInteger x) +makeScalarValue BoolField (Parser.IntValue x) | x == 0 = Right False | x == 1 = Right True | otherwise = Left $ "Unrecognized bool value " ++ show x -makeValue DoubleField (Parser.DoubleValue x) = Right x -makeValue FloatField (Parser.DoubleValue x) = Right (realToFrac x) -makeValue BoolField (Parser.EnumValue x) - | x == "true" = Right True - | x == "false" = Right False +makeScalarValue DoubleField (Parser.DoubleValue x) = Right x +makeScalarValue FloatField (Parser.DoubleValue x) = Right (realToFrac x) +makeScalarValue BoolField (Parser.EnumValue x) + | x `elem` ["true", "True", "t"] = Right True + | x `elem` ["false", "False", "f"] = Right False | otherwise = Left $ "Unrecognized bool value " ++ show x -makeValue StringField (Parser.ByteStringValue x) = Right (Text.decodeUtf8 x) -makeValue BytesField (Parser.ByteStringValue x) = Right x -makeValue EnumField (Parser.IntValue x) = +makeScalarValue StringField (Parser.ByteStringValue x) = Right (Text.decodeUtf8 x) +makeScalarValue BytesField (Parser.ByteStringValue x) = Right x +makeScalarValue EnumField (Parser.IntValue x) = maybe (Left $ "Unrecognized enum value " ++ show x) Right (maybeToEnum $ fromInteger x) -makeValue EnumField (Parser.EnumValue x) = +makeScalarValue EnumField (Parser.EnumValue x) = maybe (Left $ "Unrecognized enum value " ++ show x) Right (readEnum x) -makeValue MessageField (Parser.MessageValue x) = buildMessage x -makeValue GroupField (Parser.MessageValue x) = buildMessage x -makeValue f val = Left $ "Type mismatch parsing text format: " ++ show (f, val) +makeScalarValue f val = Left $ "Type mismatch: " ++ show (f, val) diff --git a/src/Data/ProtoLens/TextFormat/Parser.hs b/src/Data/ProtoLens/TextFormat/Parser.hs --- a/src/Data/ProtoLens/TextFormat/Parser.hs +++ b/src/Data/ProtoLens/TextFormat/Parser.hs @@ -6,6 +6,8 @@ -- | Helper utilities to parse the human-readable text format into a -- proto-agnostic syntax tree. +{-# LANGUAGE FlexibleContexts #-} + module Data.ProtoLens.TextFormat.Parser ( Message , Field(..) @@ -14,21 +16,23 @@ , parser ) where -import Data.ByteString (ByteString, pack) -import Data.Char (ord) +import Control.Applicative ((<|>), many) +import Control.Monad (liftM, liftM2, mzero, unless) +import Data.ByteString (ByteString) +import Data.ByteString.Builder (Builder, char8, charUtf8, toLazyByteString, word8) +import Data.ByteString.Lazy (toStrict) +import Data.Char (digitToInt, isSpace) +import Data.Functor (($>)) import Data.Functor.Identity (Identity) import Data.List (intercalate) -import Data.Maybe (catMaybes) +import qualified Data.Text as StrictText import Data.Text.Lazy (Text) -import Data.Word (Word8) -import Numeric (readOct, readHex) +import Text.Parsec (()) import Text.Parsec.Char (alphaNum, char, hexDigit, letter, octDigit, oneOf, satisfy) import Text.Parsec.Text.Lazy (Parser) -import Text.Parsec.Combinator (choice, eof, many1, optionMaybe, sepBy1) +import Text.Parsec.Combinator (choice, eof, many1, sepBy1) import Text.Parsec.Token hiding (octal) -import Control.Applicative ((<|>), many) -import Control.Monad (liftM, liftM2, mzero) -- | A 'TokenParser' for the protobuf text format. ptp :: GenTokenParser Text () Identity @@ -51,7 +55,6 @@ , caseSensitive = True } - type Message = [Field] data Field = Field Key Value @@ -67,13 +70,14 @@ data Value = IntValue Integer -- ^ An integer | DoubleValue Double -- ^ Any floating point number | ByteStringValue ByteString -- ^ A string or bytes literal - | MessageValue Message -- ^ A sub message + | MessageValue (Maybe StrictText.Text) Message -- ^ A sub message, with an optional type URI | EnumValue String -- ^ Any undelimited string (including false & true) deriving (Show,Ord,Eq) instance Show Key where - show (Key name) = name + show (Key name) = show name -- Quoting field names (i.e., `"field"` vs `field` + -- leads to nicer error messages. show (UnknownKey k) = show k show (ExtensionKey name) = "[" ++ intercalate "." name ++ "]" show (UnknownExtensionKey k) = "[" ++ show k ++ "]" @@ -98,11 +102,18 @@ value <- naturalOrFloat ptp return $ makeNumberValue negative value parseString = liftM (ByteStringValue . mconcat) - $ many1 $ lexeme ptp $ protoStringLiteral + $ many1 $ lexeme ptp protoStringLiteral parseEnumValue = liftM EnumValue (identifier ptp) - parseMessageValue = liftM MessageValue - (braces ptp parseMessage <|> angles ptp parseMessage) + parseMessageValue = + braces ptp (parseAny <|> + liftM (MessageValue Nothing) parseMessage) <|> + angles ptp (liftM (MessageValue Nothing) parseMessage) + typeUri = liftM StrictText.pack (many (satisfy (\c -> c /= ']' && not (isSpace c)))) + "type URI" + parseAny = liftM2 MessageValue (liftM Just (brackets ptp typeUri)) + (braces ptp parseMessage) + makeNumberValue :: Bool -> Either Integer Double -> Value makeNumberValue True (Left intValue) = IntValue (negate intValue) makeNumberValue False (Left intValue) = IntValue intValue @@ -119,39 +130,43 @@ protoStringLiteral :: Parser ByteString protoStringLiteral = do initialQuoteChar <- char '\'' <|> char '\"' - word8s <- many stringChar + let quoted = do + _ <- char '\\' + choice + [ char 'a' $> char8 '\a' + , char 'b' $> char8 '\b' + , char 'f' $> char8 '\f' + , char 'n' $> char8 '\n' + , char 'r' $> char8 '\r' + , char 't' $> char8 '\t' + , char 'v' $> char8 '\v' + , char '\\' $> char8 '\\' + , char '\'' $> char8 '\'' + , char '\"' $> char8 '\"' + , oneOf "xX" *> parse8BitToBuilder hexDigit 16 (1, 2) + , oneOf "uU" *> fail "Unicode in string literals not yet supported" + , parse8BitToBuilder octDigit 8 (1, 3) + ] + unquoted = charUtf8 <$> satisfy (/= initialQuoteChar) + builders <- many $ quoted <|> unquoted _ <- char initialQuoteChar - return $ pack word8s + return $ toStrict $ toLazyByteString $ mconcat builders where - stringChar :: Parser Word8 - stringChar = nonEscape <|> stringEscape - nonEscape = fmap (fromIntegral . ord) - $ satisfy (\c -> c `notElem` "\\\'\"" && ord c < 256) - stringEscape = char '\\' >> (octal <|> hex <|> unicode <|> simple) - octal = do d0 <- octDigit - d1 <- optionMaybe octDigit - d2 <- optionMaybe octDigit - readMaybeDigits readOct [Just d0, d1, d2] - readMaybeDigits :: ReadS Word8 -> [Maybe Char] -> Parser Word8 - readMaybeDigits reader - = return . (\str -> let [(v, "")] = reader str in v) . catMaybes - hex = do _ <- oneOf "xX" - d0 <- hexDigit - d1 <- optionMaybe hexDigit - readMaybeDigits readHex [Just d0, d1] - unicode = oneOf "uU" >> fail "Unicode in string literals not yet supported" - simple = choice $ map charRet [ ('a', '\a') - , ('b', '\b') - , ('f', '\f') - , ('n', '\n') - , ('r', '\r') - , ('t', '\t') - , ('v', '\v') - , ('\\', '\\') - , ('\'', '\'') - , ('\"', '\"') - ] - where - charRet :: (Char, Char) -> Parser Word8 - charRet (escapeCh, ch) = do _ <- char escapeCh - return $ fromIntegral $ ord ch + -- | Apply a parser between 'min' and 'max' times, failing otherwise. + manyN :: Parser a -> (Int, Int) -> Parser [a] + manyN _ (_, 0) = return [] + manyN p (0, maX) = ((:) <$> p <*> manyN p (0, maX - 1)) <|> pure [] + manyN p (miN, maX) = (:) <$> p <*> manyN p (miN - 1, maX - 1) + + -- | Parse a number in 'base' with between 'min' and 'max' digits. + parseNum :: Parser Char -> Int -> (Int, Int) -> Parser Int + parseNum p base range = do + digits <- map digitToInt <$> manyN p range + return $ foldl (\a d -> a * base + d) 0 digits + + -- | Parse a number and return a builder for the 8-bit char it represents. + parse8BitToBuilder :: Parser Char -> Int -> (Int, Int) -> Parser Builder + parse8BitToBuilder p base range = do + value <- parseNum p base range + unless (value < 256) $ fail "Escaped number is not 8-bit" + return $ word8 $ fromIntegral value diff --git a/tests/growing_test.hs b/tests/growing_test.hs new file mode 100644 --- /dev/null +++ b/tests/growing_test.hs @@ -0,0 +1,37 @@ +-- | Unit and property tests for Data.ProtoLens.Encoding.Growing. +module Main (main) where + +import Control.Monad (void) +import Control.Monad.ST +import Data.Foldable (foldlM) +import qualified Data.Vector.Unboxed as V +import Test.QuickCheck +import Test.Tasty (defaultMain, testGroup) +import Test.Tasty.QuickCheck (testProperty) + +import Data.ProtoLens.Encoding.Growing + +main :: IO () +main = defaultMain $ testGroup "tests" + [ testProperty "fromList" testFromList + , testProperty "unchanged" testUnchanged + ] + +testFromList :: [Int] -> Property +testFromList xs = fromListGrowing xs === V.fromList xs + +fromListGrowing :: V.Unbox a => [a] -> V.Vector a +fromListGrowing xs0 = runST (new >>= fill xs0 >>= unsafeFreeze) + +fill :: V.Unbox a => [a] -> Growing V.Vector s a -> ST s (Growing V.Vector s a) +fill xs v = foldlM append v xs + +-- Test a weak form of immutability: filling in more values (which may or may +-- not cause reallocations) doesn't affect the current value. +testUnchanged :: [Int] -> [Int] -> Property +testUnchanged xs ys = + let xs' = runST (do + v <- new >>= fill xs + void $ fill ys v + unsafeFreeze v) + in xs' === V.fromList xs diff --git a/tests/parser_test.hs b/tests/parser_test.hs new file mode 100644 --- /dev/null +++ b/tests/parser_test.hs @@ -0,0 +1,96 @@ +-- | Unit and property tests for our custom parsing monad. +module Main (main) where + +import qualified Data.ByteString as B +import Data.Either (isLeft) + +import Test.QuickCheck +import Test.Tasty (defaultMain, testGroup, TestTree) +import Test.Tasty.QuickCheck (testProperty) + +import Data.ProtoLens.Encoding.Bytes +import Data.ProtoLens.Encoding.Parser + +main :: IO () +main = defaultMain $ testGroup "tests" + [ testGroup "Parser" testParser + , testGroup "getWord8" testGetWord8 + , testGroup "getBytes" testGetBytes + , testGroup "getWord32le" testGetWord32le + , testGroup "failure" testFailure + , testGroup "isolate" testIsolate + ] + +testParser :: [TestTree] +testParser = + -- Test out the Applicative instance by using "traverse" to read the same number of bytes + -- as in the input. + -- "traverse (const f) g" runs f once for every element of g. + [ testProperty "traverse" $ \ws -> runParser (traverse (const getWord8) ws) + (B.pack ws) + === Right ws + ] + +testGetWord8 :: [TestTree] +testGetWord8 = + [ testProperty "atEnd" $ \ws -> runParser atEnd (B.pack ws) === Right (null ws) + , testProperty "manyTillEnd" + $ \ws -> runParser (manyTillEnd getWord8) (B.pack ws) === Right ws + ] + +testGetBytes :: [TestTree] +testGetBytes = + [ testProperty "many" + $ \ws -> let + packed = map B.pack ws + in runParser (mapM (getBytes . B.length) packed) (B.concat packed) + === Right packed + , testProperty "negative length" + $ \n ws -> n < 0 ==> counterexampleF isLeft + (runParser (getBytes n) $ B.pack ws) + ] + +testGetWord32le :: [TestTree] +testGetWord32le = + [ testProperty "align" + $ \ws -> length ws `mod` 4 /= 0 ==> + counterexampleF isLeft (runParser (manyTillEnd getWord32le) (B.pack ws)) + , testProperty "manyTillEnd" $ \ws -> + runParser (manyTillEnd getWord32le) (runBuilder $ foldMap putFixed32 ws) + === Right ws + ] + +testFailure :: [TestTree] +testFailure = + [ testProperty "fail-fast" $ \bs -> + runParser (fail "abcde") (B.pack bs) + === (Left "abcde" :: Either String ()) + , testProperty "" $ \bs -> + runParser (fail "abcde" "fghij") (B.pack bs) + === (Left "fghij: abcde" :: Either String ()) + ] + +testIsolate :: [TestTree] +testIsolate = + [ testProperty "many" $ \bs bs' -> + runParser ((,) <$> isolate (length bs) (manyTillEnd getWord8) <*> + manyTillEnd getWord8) + (B.pack (bs ++ bs')) + == Right (bs, bs') + , testProperty "negative length" $ \n ws -> + n < 0 ==> counterexampleF isLeft $ runParser (isolate n getWord8) $ B.pack ws + ] + +-- Since this is a test, just implement the slow stack-heavy way. +manyTillEnd :: Parser a -> Parser [a] +manyTillEnd p = do + end <- atEnd + if end + then return [] + else do + x <- p + xs <- manyTillEnd p + return $ x : xs + +counterexampleF :: (Testable prop, Show a) => (a -> prop) -> a -> Property +counterexampleF f x = counterexample (show x) $ f x