produce.go 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147
  1. package produce
  2. import (
  3. "fmt"
  4. "github.com/segmentio/kafka-go/protocol"
  5. )
  6. func init() {
  7. protocol.Register(&Request{}, &Response{})
  8. }
  9. type Request struct {
  10. TransactionalID string `kafka:"min=v3,max=v8,nullable"`
  11. Acks int16 `kafka:"min=v0,max=v8"`
  12. Timeout int32 `kafka:"min=v0,max=v8"`
  13. Topics []RequestTopic `kafka:"min=v0,max=v8"`
  14. }
  15. func (r *Request) ApiKey() protocol.ApiKey { return protocol.Produce }
  16. func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {
  17. broker := protocol.Broker{ID: -1}
  18. for i := range r.Topics {
  19. t := &r.Topics[i]
  20. topic, ok := cluster.Topics[t.Topic]
  21. if !ok {
  22. return broker, NewError(protocol.NewErrNoTopic(t.Topic))
  23. }
  24. for j := range t.Partitions {
  25. p := &t.Partitions[j]
  26. partition, ok := topic.Partitions[p.Partition]
  27. if !ok {
  28. return broker, NewError(protocol.NewErrNoPartition(t.Topic, p.Partition))
  29. }
  30. if b, ok := cluster.Brokers[partition.Leader]; !ok {
  31. return broker, NewError(protocol.NewErrNoLeader(t.Topic, p.Partition))
  32. } else if broker.ID < 0 {
  33. broker = b
  34. } else if b.ID != broker.ID {
  35. return broker, NewError(fmt.Errorf("mismatching leaders (%d!=%d)", b.ID, broker.ID))
  36. }
  37. }
  38. }
  39. return broker, nil
  40. }
  41. func (r *Request) Prepare(apiVersion int16) {
  42. // Determine which version of the message should be used, based on which
  43. // version of the Produce API is supported by the server.
  44. //
  45. // In version 0.11, kafka gives this error:
  46. //
  47. // org.apache.kafka.common.record.InvalidRecordException
  48. // Produce requests with version 3 are only allowed to contain record batches with magic version.
  49. //
  50. // In version 2.x, kafka refuses the message claiming that the CRC32
  51. // checksum is invalid.
  52. var recordVersion int8
  53. if apiVersion < 3 {
  54. recordVersion = 1
  55. } else {
  56. recordVersion = 2
  57. }
  58. for i := range r.Topics {
  59. t := &r.Topics[i]
  60. for j := range t.Partitions {
  61. p := &t.Partitions[j]
  62. // Allow the program to overload the version if really needed.
  63. if p.RecordSet.Version == 0 {
  64. p.RecordSet.Version = recordVersion
  65. }
  66. }
  67. }
  68. }
  69. func (r *Request) HasResponse() bool {
  70. return r.Acks != 0
  71. }
  72. type RequestTopic struct {
  73. Topic string `kafka:"min=v0,max=v8"`
  74. Partitions []RequestPartition `kafka:"min=v0,max=v8"`
  75. }
  76. type RequestPartition struct {
  77. Partition int32 `kafka:"min=v0,max=v8"`
  78. RecordSet protocol.RecordSet `kafka:"min=v0,max=v8"`
  79. }
  80. type Response struct {
  81. Topics []ResponseTopic `kafka:"min=v0,max=v8"`
  82. ThrottleTimeMs int32 `kafka:"min=v1,max=v8"`
  83. }
  84. func (r *Response) ApiKey() protocol.ApiKey { return protocol.Produce }
  85. type ResponseTopic struct {
  86. Topic string `kafka:"min=v0,max=v8"`
  87. Partitions []ResponsePartition `kafka:"min=v0,max=v8"`
  88. }
  89. type ResponsePartition struct {
  90. Partition int32 `kafka:"min=v0,max=v8"`
  91. ErrorCode int16 `kafka:"min=v0,max=v8"`
  92. BaseOffset int64 `kafka:"min=v0,max=v8"`
  93. LogAppendTime int64 `kafka:"min=v2,max=v8"`
  94. LogStartOffset int64 `kafka:"min=v5,max=v8"`
  95. RecordErrors []ResponseError `kafka:"min=v8,max=v8"`
  96. ErrorMessage string `kafka:"min=v8,max=v8,nullable"`
  97. }
  98. type ResponseError struct {
  99. BatchIndex int32 `kafka:"min=v8,max=v8"`
  100. BatchIndexErrorMessage string `kafka:"min=v8,max=v8,nullable"`
  101. }
  102. var (
  103. _ protocol.BrokerMessage = (*Request)(nil)
  104. _ protocol.PreparedMessage = (*Request)(nil)
  105. )
  106. type Error struct {
  107. Err error
  108. }
  109. func NewError(err error) *Error {
  110. return &Error{Err: err}
  111. }
  112. func (e *Error) Error() string {
  113. return fmt.Sprintf("fetch request error: %v", e.Err)
  114. }
  115. func (e *Error) Unwrap() error {
  116. return e.Err
  117. }