0

I'm trying to extract Protobuf custom options from a FileDescriptorSet generated by the protoc compiler. I'm unable to do so using protoreflect. So, I tried to do so using the protojson library.

PS : Importing the Go-generated code is not an option for my use case.

Here's the Protobuf Message I'm testing with :

syntax = "proto3";

option go_package = "./protoze";

import "google/protobuf/descriptor.proto";

extend google.protobuf.FieldOptions {
    string Meta = 50000;
}
extend google.protobuf.FileOptions {
    string Food = 50001;
}
option (Food) = "cheese";
message X {
 int64 num = 1;
}
message P {
    string Fname          = 1 [json_name = "FNAME"];
    string Lname          = 2 [json_name = "0123", (Meta) = "Yo"]; 
    string Designation    = 3;
    repeated string Email = 4;
    string UserID         = 5;
    string EmpID          = 6;
    repeated X z          = 7;
}
// protoc --go_out=. filename.proto

Here's how far I got :

package main

import (
    "fmt"
    "io/ioutil"
    "os/exec"

    "google.golang.org/protobuf/encoding/protojson"
    "google.golang.org/protobuf/proto"
    "google.golang.org/protobuf/types/descriptorpb"
)

func main() {
    exec.Command("protoc", "-oBinaryFile", "1.proto").Run()
    Fset := descriptorpb.FileDescriptorSet{}
    byts, _ := ioutil.ReadFile("File")
    proto.Unmarshal(byts, &Fset)
    byts, _ = protojson.Marshal(Fset.File[0])
    fmt.Println(string(byts))
}

And here's the output JSON

{
  "name": "1.proto",
  "dependency": [
    "google/protobuf/descriptor.proto"
  ],
  "messageType": [
    {
      "name": "X",
      "field": [
        {
          "name": "num",
          "number": 1,
          "label": "LABEL_OPTIONAL",
          "type": "TYPE_INT64",
          "jsonName": "num"
        }
      ]
    },
    {
      "name": "P",
      "field": [
        {
          "name": "Fname",
          "number": 1,
          "label": "LABEL_OPTIONAL",
          "type": "TYPE_STRING",
          "jsonName": "FNAME"
        },
        {
          "name": "Lname",
          "number": 2,
          "label": "LABEL_OPTIONAL",
          "type": "TYPE_STRING",
          "jsonName": "0123",
          "options": {}
        },
        {
          "name": "Designation",
          "number": 3,
          "label": "LABEL_OPTIONAL",
          "type": "TYPE_STRING",
          "jsonName": "Designation"
        },
        {
          "name": "Email",
          "number": 4,
          "label": "LABEL_REPEATED",
          "type": "TYPE_STRING",
          "jsonName": "Email"
        },
        {
          "name": "UserID",
          "number": 5,
          "label": "LABEL_OPTIONAL",
          "type": "TYPE_STRING",
          "jsonName": "UserID"
        },
        {
          "name": "EmpID",
          "number": 6,
          "label": "LABEL_OPTIONAL",
          "type": "TYPE_STRING",
          "jsonName": "EmpID"
        },
        {
          "name": "z",
          "number": 7,
          "label": "LABEL_REPEATED",
          "type": "TYPE_MESSAGE",
          "typeName": ".X",
          "jsonName": "z"
        }
      ]
    }
  ],
  "extension": [
    {
      "name": "Meta",
      "number": 50000,
      "label": "LABEL_OPTIONAL",
      "type": "TYPE_STRING",
      "extendee": ".google.protobuf.FieldOptions",
      "jsonName": "Meta"
    },
    {
      "name": "Food",
      "number": 50001,
      "label": "LABEL_OPTIONAL",
      "type": "TYPE_STRING",
      "extendee": ".google.protobuf.FileOptions",
      "jsonName": "Food"
    }
  ],
  "options": {
    "goPackage": "./protoze"
  },
  "syntax": "proto3"
}

So, data about my custom options showed up in the extensions. But what I really wanted was the value of those Custom Options in the "options" as well. (Which in my case was (Food) = "Cheese" and I want Cheese)

Can someone tell me how I can extract my custom options from the FileDescriptorSet using Protoreflect or by using Protojson.

I tried a lot to try and extract it using Protoreflect but failed !

mujtaba1747
  • 33
  • 1
  • 6

1 Answers1

1

Although not specifically an answer to how to get the custom options in a generated JSON, I believe I have an answer to what sounds like your underlying question: how to access the custom options without loading the generated Go code. This is thanks to dsnet's answer to my question on the golang issues board. Needless to say all the credit for this tricky solution goes to him. The punchline is to Marshal and then Unmarshal the options using a runtime-populated protoregistry.Types that actually knows about the custom options.

I made a complete demonstration of this approach working in this repo, and the key section (all the guts of which come from dsnet's example) is here:

func main() {
    protogen.Options{
    }.Run(func(gen *protogen.Plugin) error {

        gen.SupportedFeatures = uint64(pluginpb.CodeGeneratorResponse_FEATURE_PROTO3_OPTIONAL)

        // The type information for all extensions is in the source files,
        // so we need to extract them into a dynamically created protoregistry.Types.
        extTypes := new(protoregistry.Types)
        for _, file := range gen.Files {
            if err := registerAllExtensions(extTypes, file.Desc); err != nil {
                panic(err)
            }
        }

        // run through the files again, extracting and printing the Message options
        for _, sourceFile := range gen.Files {
            if !sourceFile.Generate {
                continue
            }

            // setup output file
            outputfile := gen.NewGeneratedFile("./out.txt", sourceFile.GoImportPath)

            for _, message := range sourceFile.Messages {
                outputfile.P(fmt.Sprintf("\nMessage %s:", message.Desc.Name()))

                // The MessageOptions as provided by protoc does not know about
                // dynamically created extensions, so they are left as unknown fields.
                // We round-trip marshal and unmarshal the options with
                // a dynamically created resolver that does know about extensions at runtime.
                options := message.Desc.Options().(*descriptorpb.MessageOptions)
                b, err := proto.Marshal(options)
                if err != nil {
                    panic(err)
                }
                options.Reset()
                err = proto.UnmarshalOptions{Resolver: extTypes}.Unmarshal(b, options)
                if err != nil {
                    panic(err)
                }

                // Use protobuf reflection to iterate over all the extension fields,
                // looking for the ones that we are interested in.
                options.ProtoReflect().Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
                    if !fd.IsExtension() {
                        return true
                    }

                    outputfile.P(fmt.Sprintf("Value of option %s is %s",fd.Name(), v.String()))

                    // Make use of fd and v based on their reflective properties.

                    return true
                })
            }
        }
        
        return nil
    })
}

// Recursively register all extensions into the provided protoregistry.Types,
// starting with the protoreflect.FileDescriptor and recursing into its MessageDescriptors,
// their nested MessageDescriptors, and so on.
//
// This leverages the fact that both protoreflect.FileDescriptor and protoreflect.MessageDescriptor 
// have identical Messages() and Extensions() functions in order to recurse through a single function
func registerAllExtensions(extTypes *protoregistry.Types, descs interface {
    Messages() protoreflect.MessageDescriptors
    Extensions() protoreflect.ExtensionDescriptors
}) error {
    mds := descs.Messages()
    for i := 0; i < mds.Len(); i++ {
        registerAllExtensions(extTypes, mds.Get(i))
    }
    xds := descs.Extensions()
    for i := 0; i < xds.Len(); i++ {
        if err := extTypes.RegisterExtension(dynamicpb.NewExtensionType(xds.Get(i))); err != nil {
            return err
        }
    }
    return nil
}
tgorton
  • 11
  • 2