Support protobuf editions and the protoc-gen-go Opaque API

.github/workflows/ci.yml

@@ -36,6 +36,8 @@ jobs:
 
     - run: go vet ./...
 
+    - run: go vet -tags protoopaque ./testproto/hybrid/...
+
     - run: make genall
     - run: git --no-pager diff --exit-code
 

Makefile

@@ -66,6 +66,26 @@ gen-testproto: get-grpc-testproto gen-wkt-testproto install
 		testproto/unsafe/unsafe.proto \
 		testproto/unique/unique.proto \
 		|| exit 1;
+	$(PROTOBUF_ROOT)/src/protoc \
+		--proto_path=testproto \
+		--proto_path=include \
+		--go_out=. --plugin protoc-gen-go="${GOBIN}/protoc-gen-go" \
+		--go_opt=default_api_level=API_OPAQUE \
+		--go-vtproto_out=allow-empty=true:. --plugin protoc-gen-go-vtproto="${GOBIN}/protoc-gen-go-vtproto" \
+		--go-vtproto_opt=default_api_level=API_OPAQUE \
+		-I$(PROTOBUF_ROOT)/src \
+		testproto/opaque/opaque.proto \
+		|| exit 1;
+	$(PROTOBUF_ROOT)/src/protoc \
+		--proto_path=testproto \
+		--proto_path=include \
+		--go_out=. --plugin protoc-gen-go="${GOBIN}/protoc-gen-go" \
+		--go_opt=default_api_level=API_HYBRID \
+		--go-vtproto_out=allow-empty=true:. --plugin protoc-gen-go-vtproto="${GOBIN}/protoc-gen-go-vtproto" \
+		--go-vtproto_opt=default_api_level=API_HYBRID \
+		-I$(PROTOBUF_ROOT)/src \
+		testproto/hybrid/hybrid.proto \
+		|| exit 1;
 	$(PROTOBUF_ROOT)/src/protoc \
 		--proto_path=testproto \
 		--proto_path=include \
@@ -77,6 +97,20 @@ gen-testproto: get-grpc-testproto gen-wkt-testproto install
 		testproto/empty/empty.proto \
 		|| exit 1;
 
+# Not part of genall: the output is checked in because the vendored protoc 21.12
+# predates editions. Regenerating needs PROTOC pointed at protoc >= 32.
+PROTOC ?= protoc
+
+gen-editions-testproto: install
+	$(PROTOC) \
+		--proto_path=testproto \
+		--proto_path=include \
+		--go_out=. --plugin protoc-gen-go="${GOBIN}/protoc-gen-go" \
+		--go-vtproto_out=allow-empty=true:. --plugin protoc-gen-go-vtproto="${GOBIN}/protoc-gen-go-vtproto" \
+		testproto/editions/editions.proto \
+		testproto/editions/editions2023.proto \
+		|| exit 1;
+
 get-grpc-testproto: install
 	$(PROTOBUF_ROOT)/src/protoc \
 		--proto_path=. \

README.md

@@ -146,6 +146,20 @@ message Label {
 
 9. (Optional) Switch your RPC framework to use the optimized helpers (see following sections)
 
+## Editions and API levels
+
+`vtprotobuf` accepts `proto2`, `proto3` and editions 2023 and 2024 input, and generates code for all three `protoc-gen-go` API levels.
+
+- **Pass the same `default_api_level` (and any `apilevelM<file>`) to both plugins.** `protoc-gen-go` and `protoc-gen-go-vtproto` each resolve the API level independently; if they disagree, `vtprotobuf` generates open-layout code against an opaque struct and the package fails to compile.
+
+- **Hybrid** files get a `//go:build !protoopaque` constraint, matching the variant `protoc-gen-go` generates without that tag. Under `-tags protoopaque` the `vtprotobuf` helpers are absent rather than broken, so guard their use with a type assertion as described for `buildTag` above.
+
+- **`wrap=true` is incompatible with the opaque API**, which does not export the struct fields the wrapper package would have to reach; it is rejected.
+
+- **`[lazy = true]` is rejected.** `vtprotobuf` cannot honour the lazy unmarshalling contract, and silently ignoring it would drop fields.
+
+- Under the opaque API, reading or writing a field with explicit presence costs one atomic operation on the presence bitfield.
+
 ## `vtprotobuf` package and well-known types
 
 Your generated `_vtproto.pb.go` files will have a dependency on this Go package to access some helper functions as well as the optimized code for ProtoBuf [well-known types](https://protobuf.dev/reference/protobuf/google.protobuf/). `vtprotobuf` will detect these types embedded in your own Messages and generate optimized code to marshal and unmarshal them.

features/clone/clone.go

@@ -39,27 +39,25 @@ func (p *clone) Name() string {
 }
 
 func (p *clone) GenerateFile(file *protogen.File) bool {
-	proto3 := file.Desc.Syntax() == protoreflect.Proto3
-
 	for _, message := range file.Messages {
-		p.processMessage(proto3, message)
+		p.processMessage(message)
 	}
 
 	return p.once
 }
 
 // cloneOneofField generates the statements for cloning a oneof field
 func (p *clone) cloneOneofField(lhsBase, rhsBase string, oneof *protogen.Oneof) {
-	fieldname := oneof.GoName
+	fieldname := p.OneofName(oneof)
 	ccInterfaceName := "is" + oneof.GoIdent.GoName
 	lhs := lhsBase + "." + fieldname
 	rhs := rhsBase + "." + fieldname
 	p.P(`if `, rhs, ` != nil {`)
 	if p.IsWellKnownType(oneof.Parent) {
 		p.P(`switch c := `, rhs, `.(type) {`)
 		for _, f := range oneof.Fields {
-			p.P(`case *`, f.GoIdent, `:`)
-			p.P(lhs, `= (*`, f.GoIdent, `)((*`, p.WellKnownFieldMap(f), `)(c).`, cloneName, `())`)
+			p.P(`case *`, p.OneofWrapperIdent(f), `:`)
+			p.P(lhs, `= (*`, p.OneofWrapperIdent(f), `)((*`, p.WellKnownFieldMap(f), `)(c).`, cloneName, `())`)
 		}
 		p.P(`}`)
 	} else {
@@ -98,21 +96,26 @@ func (p *clone) cloneFieldSingular(lhs, rhs string, kind protoreflect.Kind, mess
 }
 
 // cloneField generates the code for cloning a field in a protobuf.
-func (p *clone) cloneField(lhsBase, rhsBase string, allFieldsNullable bool, field *protogen.Field) {
+func (p *clone) cloneField(lhsBase, rhsBase string, wrapper bool, field *protogen.Field) {
 	// At this point, if we encounter a non-synthetic oneof, we assume it to be the representative
 	// field for that oneof.
 	if field.Oneof != nil && !field.Oneof.Desc.IsSynthetic() {
 		p.cloneOneofField(lhsBase, rhsBase, field.Oneof)
 		return
 	}
 
-	if !isReference(allFieldsNullable, field) {
+	pointer := !wrapper && p.FieldStorageIsPointer(field)
+	if !isReference(pointer, field) {
 		panic("method should not be invoked for non-reference fields")
 	}
 
 	fieldname := field.GoName
-	lhs := lhsBase + "." + fieldname
 	rhs := rhsBase + "." + fieldname
+	if !wrapper {
+		fieldname = p.FieldName(field)
+		rhs = p.FieldSliceExpr(rhsBase, field)
+	}
+	lhs := lhsBase + "." + fieldname
 
 	// At this point, we are only looking at reference types (pointers, maps, slices, interfaces), which can all
 	// be nil.
@@ -141,7 +144,11 @@ func (p *clone) cloneField(lhsBase, rhsBase string, allFieldsNullable bool, fiel
 			p.cloneFieldSingular("tmpContainer[k]", "v", fieldKind, msg)
 			p.P(`}`)
 		}
-		p.P(lhs, ` = tmpContainer`)
+		if p.FieldStorageIsPointer(field) {
+			p.P(lhs, ` = &tmpContainer`)
+		} else {
+			p.P(lhs, ` = tmpContainer`)
+		}
 	} else if isScalar(fieldKind) {
 		p.P(`tmpVal := *`, rhs)
 		p.P(lhs, ` = &tmpVal`)
@@ -151,10 +158,10 @@ func (p *clone) cloneField(lhsBase, rhsBase string, allFieldsNullable bool, fiel
 	p.P(`}`)
 }
 
-func (p *clone) generateCloneMethodsForMessage(proto3 bool, message *protogen.Message) {
+func (p *clone) generateCloneMethodsForMessage(message *protogen.Message) {
 	ccTypeName := message.GoIdent.GoName
 	p.P(`func (m *`, ccTypeName, `) `, cloneName, `() *`, ccTypeName, ` {`)
-	p.body(!proto3, ccTypeName, message)
+	p.body(ccTypeName, message)
 	p.P(`}`)
 	p.P()
 
@@ -169,7 +176,7 @@ func (p *clone) generateCloneMethodsForMessage(proto3 bool, message *protogen.Me
 // body generates the code for the actual cloning logic of a structure containing the given fields.
 // In practice, those can be the fields of a message.
 // The object to be cloned is assumed to be called "m".
-func (p *clone) body(allFieldsNullable bool, ccTypeName string, message *protogen.Message) {
+func (p *clone) body(ccTypeName string, message *protogen.Message) {
 	// The method body for a message or a oneof wrapper always starts with a nil check.
 	p.P(`if m == nil {`)
 	// We use an explicitly typed nil to avoid returning the nil interface in the oneof wrapper
@@ -196,19 +203,20 @@ func (p *clone) body(allFieldsNullable bool, ccTypeName string, message *protoge
 			continue
 		}
 
-		if !isReference(allFieldsNullable, field) {
-			p.P(`r.`, field.GoName, ` = m.`, field.GoName)
+		fieldname := p.FieldName(field)
+		if !isReference(p.FieldStorageIsPointer(field), field) {
+			p.P(`r.`, fieldname, ` = m.`, fieldname)
 			continue
 		}
 		// Shortcut: for types where we know that an optimized clone method exists, we can call it directly as it is
 		// nil-safe.
 		if field.Desc.Cardinality() != protoreflect.Repeated {
 			switch {
 			case p.IsWellKnownType(field.Message):
-				p.P(`r.`, field.GoName, ` = (*`, field.Message.GoIdent, `)((*`, p.WellKnownTypeMap(field.Message), `)(m.`, field.GoName, `).`, cloneName, `())`)
+				p.P(`r.`, fieldname, ` = (*`, field.Message.GoIdent, `)((*`, p.WellKnownTypeMap(field.Message), `)(m.`, fieldname, `).`, cloneName, `())`)
 				continue
 			case p.IsLocalMessage(field.Message):
-				p.P(`r.`, field.GoName, ` = m.`, field.GoName, `.`, cloneName, `()`)
+				p.P(`r.`, fieldname, ` = m.`, fieldname, `.`, cloneName, `()`)
 				continue
 			}
 		}
@@ -217,9 +225,11 @@ func (p *clone) body(allFieldsNullable bool, ccTypeName string, message *protoge
 
 	// Generate explicit assignment statements for all reference fields.
 	for _, field := range refFields {
-		p.cloneField("r", "m", allFieldsNullable, field)
+		p.cloneField("r", "m", false, field)
 	}
 
+	p.CopyPresence("r", "m", message)
+
 	if !p.Wrapper() && !p.ShouldIgnoreUnknownFields(message) {
 		// Clone unknown fields, if any
 		p.P(`if len(m.unknownFields) > 0 {`)
@@ -262,15 +272,15 @@ func (p *clone) bodyForOneOf(ccTypeName string, field *protogen.Field) {
 	}
 
 	// Generate explicit assignment statements for reference field.
-	p.cloneField("r", "m", false, field)
+	p.cloneField("r", "m", true, field)
 
 	p.P(`return r`)
 }
 
 // generateCloneMethodsForOneof generates the clone method for the oneof wrapper type of a
 // field in a oneof.
 func (p *clone) generateCloneMethodsForOneof(message *protogen.Message, field *protogen.Field) {
-	ccTypeName := field.GoIdent.GoName
+	ccTypeName := p.OneofWrapperIdent(field).GoName
 	ccInterfaceName := "is" + field.Oneof.GoIdent.GoName
 	if p.IsWellKnownType(message) {
 		p.P(`func (m *`, ccTypeName, `) `, cloneName, `() *`, ccTypeName, ` {`)
@@ -296,9 +306,9 @@ func (p *clone) processMessageOneofs(message *protogen.Message) {
 	}
 }
 
-func (p *clone) processMessage(proto3 bool, message *protogen.Message) {
+func (p *clone) processMessage(message *protogen.Message) {
 	for _, nested := range message.Messages {
-		p.processMessage(proto3, nested)
+		p.processMessage(nested)
 	}
 
 	if message.Desc.IsMapEntry() {
@@ -307,13 +317,14 @@ func (p *clone) processMessage(proto3 bool, message *protogen.Message) {
 
 	p.once = true
 
-	p.generateCloneMethodsForMessage(proto3, message)
+	p.generateCloneMethodsForMessage(message)
 	p.processMessageOneofs(message)
 }
 
 // isReference checks whether the Go equivalent of the given field is of reference type, i.e., can be nil.
-func isReference(allFieldsNullable bool, field *protogen.Field) bool {
-	if allFieldsNullable || field.Oneof != nil || field.Message != nil || field.Desc.Cardinality() == protoreflect.Repeated || field.Desc.Kind() == protoreflect.BytesKind {
+func isReference(pointer bool, field *protogen.Field) bool {
+	realOneof := field.Oneof != nil && !field.Oneof.Desc.IsSynthetic()
+	if pointer || realOneof || field.Message != nil || field.Desc.Cardinality() == protoreflect.Repeated || field.Desc.Kind() == protoreflect.BytesKind {
 		return true
 	}
 	if !isScalar(field.Desc.Kind()) {

features/equal/equal.go

@@ -32,19 +32,18 @@ var _ generator.FeatureGenerator = (*equal)(nil)
 func (p *equal) Name() string { return "equal" }
 
 func (p *equal) GenerateFile(file *protogen.File) bool {
-	proto3 := file.Desc.Syntax() == protoreflect.Proto3
 	for _, message := range file.Messages {
-		p.message(proto3, message)
+		p.message(message)
 	}
 	return p.once
 }
 
 const equalName = "EqualVT"
 const equalMessageName = "EqualMessageVT"
 
-func (p *equal) message(proto3 bool, message *protogen.Message) {
+func (p *equal) message(message *protogen.Message) {
 	for _, nested := range message.Messages {
-		p.message(proto3, nested)
+		p.message(nested)
 	}
 
 	if message.Desc.IsMapEntry() {
@@ -74,7 +73,7 @@ func (p *equal) message(proto3 bool, message *protogen.Message) {
 				continue
 			}
 
-			fieldname := field.Oneof.GoName
+			fieldname := p.OneofName(field.Oneof)
 			if _, ok := oneofs[fieldname]; ok {
 				continue
 			}
@@ -90,7 +89,7 @@ func (p *equal) message(proto3 bool, message *protogen.Message) {
 			if p.IsWellKnownType(message) {
 				p.P(`switch c := this.`, fieldname, `.(type) {`)
 				for _, f := range field.Oneof.Fields {
-					p.P(`case *`, f.GoIdent, `:`)
+					p.P(`case *`, p.OneofWrapperIdent(f), `:`)
 					p.P(`if !(*`, p.WellKnownFieldMap(f), `)(c).`, equalName, `(that.`, fieldname, `) {`)
 					p.P(`return false`)
 					p.P(`}`)
@@ -107,7 +106,7 @@ func (p *equal) message(proto3 bool, message *protogen.Message) {
 
 	for _, field := range message.Fields {
 		oneof := field.Oneof != nil && !field.Oneof.Desc.IsSynthetic()
-		nullable := field.Message != nil || (field.Oneof != nil && field.Oneof.Desc.IsSynthetic()) || (!proto3 && !oneof)
+		nullable := field.Message != nil || field.Desc.HasPresence()
 		if !oneof {
 			p.field(field, nullable)
 		}
@@ -141,7 +140,7 @@ func (p *equal) message(proto3 bool, message *protogen.Message) {
 }
 
 func (p *equal) oneof(field *protogen.Field) {
-	ccTypeName := field.GoIdent.GoName
+	ccTypeName := p.OneofWrapperIdent(field).GoName
 	ccInterfaceName := fmt.Sprintf("is%s", field.Oneof.GoIdent.GoName)
 	fieldname := field.GoName
 
@@ -188,11 +187,17 @@ func (p *equal) oneof(field *protogen.Field) {
 }
 
 func (p *equal) field(field *protogen.Field, nullable bool) {
-	fieldname := field.GoName
-
 	repeated := field.Desc.Cardinality() == protoreflect.Repeated
-	lhs := fmt.Sprintf("this.%s", fieldname)
-	rhs := fmt.Sprintf("that.%s", fieldname)
+	lhs := p.FieldSliceExpr("this", field)
+	rhs := p.FieldSliceExpr("that", field)
+
+	if !repeated && p.UsesPresenceBit(field) && !p.FieldStorageIsPointer(field) {
+		// Unset and set-to-zero differ only in the presence bit.
+		p.P(`if `, p.FieldPresent("this", field), ` != `, p.FieldPresent("that", field), ` {`)
+		p.P(`return false`)
+		p.P(`}`)
+		nullable = false
+	}
 
 	if repeated {
 		p.P(`if len(`, lhs, `) != len(`, rhs, `) {`)