module-transaction: Add support for binary conversations

A module can now initiate a binary conversation decoding the native
pointer value as it wants.

Added tests to verify the main cases

cmd/pam-moduler/tests/integration-tester-module/integration-tester-module.go

@@ -60,6 +60,9 @@ func (m *integrationTesterModule) handleRequest(authReq *authRequest, r *Request
 		case SerializableStringConvRequest:
 			args = append(args, reflect.ValueOf(
 				pam.NewStringConvRequest(v.Style, v.Request)))
+		case SerializableBinaryConvRequest:
+			args = append(args, reflect.ValueOf(
+				pam.NewBinaryConvRequestFromBytes(v.Request)))
 		default:
 			if arg == nil {
 				args = append(args, reflect.Zero(method.Type().In(i)))
@@ -76,6 +79,12 @@ func (m *integrationTesterModule) handleRequest(authReq *authRequest, r *Request
 		case pam.StringConvResponse:
 			res.ActionArgs = append(res.ActionArgs,
 				SerializableStringConvResponse{value.Style(), value.Response()})
+		case pam.BinaryConvResponse:
+			data, err := value.Decode(utils.TestBinaryDataDecoder)
+			if err != nil {
+				return nil, err
+			}
+			res.ActionArgs = append(res.ActionArgs, SerializableBinaryConvResponse{data})
 		case pam.Error:
 			authReq.lastError = value
 			res.ActionArgs = append(res.ActionArgs, value)

cmd/pam-moduler/tests/integration-tester-module/integration-tester-module_test.go

@@ -71,6 +71,21 @@ func ensureEnv(tx *pam.Transaction, variable string, expected string) error {
 	return nil
 }
 
+func (r *Request) toBytes(t *testing.T) []byte {
+	t.Helper()
+	bytes, err := r.GOB()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+		return nil
+	}
+	return bytes
+}
+
+func (r *Request) toTransactionData(t *testing.T) []byte {
+	t.Helper()
+	return utils.TestBinaryDataEncoder(r.toBytes(t))
+}
+
 func Test_Moduler_IntegrationTesterModule(t *testing.T) {
 	t.Parallel()
 	if !pam.CheckPamHasStartConfdir() {
@@ -818,6 +833,104 @@ func Test_Moduler_IntegrationTesterModule(t *testing.T) {
 				},
 			},
 		},
+		"start-conv-binary": {
+			credentials: utils.NewBinaryTransactionWithData([]byte(
+				"\x00This is a binary data request\xC5\x00\xffYes it is!"),
+				[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99}),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{
+						utils.TestBinaryDataEncoder(
+							[]byte("\x00This is a binary data request\xC5\x00\xffYes it is!")),
+					}),
+					exp: []interface{}{SerializableBinaryConvResponse{
+						[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+					}, nil},
+				},
+				{
+					r: NewRequest("StartBinaryConv",
+						utils.TestBinaryDataEncoder(
+							[]byte("\x00This is a binary data request\xC5\x00\xffYes it is!"))),
+					exp: []interface{}{SerializableBinaryConvResponse{
+						[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+					}, nil},
+				},
+			},
+		},
+		"start-conv-binary-handle-failure-passed-data-mismatch": {
+			expectedError: pam.ErrConv,
+			credentials: utils.NewBinaryTransactionWithData([]byte(
+				"\x00This is a binary data request\xC5\x00\xffYes it is!"),
+				[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99}),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{
+						(&Request{"Not the expected binary data", nil}).toTransactionData(t),
+					}),
+					exp: []interface{}{nil, pam.ErrConv},
+				},
+				{
+					r: NewRequest("StartBinaryConv",
+						(&Request{"Not the expected binary data", nil}).toTransactionData(t)),
+					exp: []interface{}{nil, pam.ErrConv},
+				},
+			},
+		},
+		"start-conv-binary-handle-failure-returned-data-mismatch": {
+			expectedError: pam.ErrConv,
+			credentials: utils.NewBinaryTransactionWithRandomData(100,
+				[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99}),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{
+						(&Request{"Wrong binary data", nil}).toTransactionData(t),
+					}),
+					exp: []interface{}{nil, pam.ErrConv},
+				},
+				{
+					r: NewRequest("StartBinaryConv",
+						(&Request{"Wrong binary data", nil}).toTransactionData(t)),
+					exp: []interface{}{nil, pam.ErrConv},
+				},
+			},
+		},
+		"start-conv-binary-in-nil": {
+			credentials: utils.NewBinaryTransactionWithData(nil,
+				(&Request{"Binary data", []interface{}{true, 123, 0.5, "yay!"}}).toBytes(t)),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{}),
+					exp: []interface{}{SerializableBinaryConvResponse{
+						(&Request{"Binary data", []interface{}{true, 123, 0.5, "yay!"}}).toBytes(t),
+					}, nil},
+				},
+				{
+					r: NewRequest("StartBinaryConv", nil),
+					exp: []interface{}{SerializableBinaryConvResponse{
+						(&Request{"Binary data", []interface{}{true, 123, 0.5, "yay!"}}).toBytes(t),
+					}, nil},
+				},
+			},
+		},
+		"start-conv-binary-out-nil": {
+			credentials: utils.NewBinaryTransactionWithData([]byte(
+				"\x00This is a binary data request\xC5\x00\xffGimme nil!"), nil),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{
+						utils.TestBinaryDataEncoder(
+							[]byte("\x00This is a binary data request\xC5\x00\xffGimme nil!")),
+					}),
+					exp: []interface{}{SerializableBinaryConvResponse{}, nil},
+				},
+				{
+					r: NewRequest("StartBinaryConv",
+						utils.TestBinaryDataEncoder(
+							[]byte("\x00This is a binary data request\xC5\x00\xffGimme nil!"))),
+					exp: []interface{}{SerializableBinaryConvResponse{}, nil},
+				},
+			},
+		},
 	}
 
 	for name, tc := range tests {
@@ -831,6 +944,13 @@ func Test_Moduler_IntegrationTesterModule(t *testing.T) {
 					Args: []string{socketPath}},
 			})
 
+			switch tc.credentials.(type) {
+			case pam.BinaryConversationHandler:
+				if !pam.CheckPamHasBinaryProtocol() {
+					t.Skip("Binary protocol is not supported")
+				}
+			}
+
 			tx, err := pam.StartConfDir(name, tc.user, tc.credentials, ts.WorkDir())
 			if err != nil {
 				t.Fatalf("start #error: %v", err)
@@ -1085,6 +1205,15 @@ func Test_Moduler_IntegrationTesterModule_Authenticate(t *testing.T) {
 				exp: []interface{}{nil, pam.ErrSystem},
 			}},
 		},
+		"StartConv-Binary": {
+			expectedError: pam.ErrSystem,
+			checkedRequests: []checkedRequest{{
+				r: NewRequest("StartConv", SerializableBinaryConvRequest{
+					[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+				}),
+				exp: []interface{}{nil, pam.ErrSystem},
+			}},
+		},
 	}
 
 	for name, tc := range tests {

cmd/pam-moduler/tests/integration-tester-module/serialization.go

@@ -36,6 +36,16 @@ type SerializableStringConvResponse struct {
 	Response string
 }
 
+// SerializableBinaryConvRequest is a serializable binary request.
+type SerializableBinaryConvRequest struct {
+	Request []byte
+}
+
+// SerializableBinaryConvResponse is a serializable binary response.
+type SerializableBinaryConvResponse struct {
+	Response []byte
+}
+
 func init() {
 	gob.Register(map[string]string{})
 	gob.Register(Request{})
@@ -49,5 +59,9 @@ func init() {
 		SerializableStringConvRequest{})
 	gob.RegisterName("main.SerializableStringConvResponse",
 		SerializableStringConvResponse{})
+	gob.RegisterName("main.SerializableBinaryConvRequest",
+		SerializableBinaryConvRequest{})
+	gob.RegisterName("main.SerializableBinaryConvResponse",
+		SerializableBinaryConvResponse{})
 	gob.Register(utils.SerializableError{})
 }

cmd/pam-moduler/tests/internal/utils/test-utils.go

@@ -1,9 +1,16 @@
 // Package utils contains the internal test utils
 package utils
 
+//#include <stdint.h>
+import "C"
+
 import (
+	"crypto/rand"
+	"encoding/binary"
 	"errors"
 	"fmt"
+	"reflect"
+	"unsafe"
 
 	"github.com/msteinert/pam/v2"
 )
@@ -167,3 +174,90 @@ func (c Credentials) RespondPAM(s pam.Style, msg string) (string, error) {
 	return "", errors.Join(pam.ErrConv,
 		&SerializableError{fmt.Sprintf("unhandled style: %v", s)})
 }
+
+// BinaryTransaction represents a binary PAM transaction handler struct.
+type BinaryTransaction struct {
+	data         []byte
+	ExpectedNull bool
+	ReturnedData []byte
+}
+
+// TestBinaryDataEncoder encodes a test binary data.
+func TestBinaryDataEncoder(bytes []byte) []byte {
+	if len(bytes) > 0xff {
+		panic("Binary transaction size not supported")
+	}
+
+	if bytes == nil {
+		return bytes
+	}
+
+	data := make([]byte, 0, len(bytes)+1)
+	data = append(data, byte(len(bytes)))
+	data = append(data, bytes...)
+	return data
+}
+
+// TestBinaryDataDecoder decodes a test binary data.
+func TestBinaryDataDecoder(ptr pam.BinaryPointer) ([]byte, error) {
+	if ptr == nil {
+		return nil, nil
+	}
+
+	length := uint8(*((*C.uint8_t)(ptr)))
+	if length == 0 {
+		return []byte{}, nil
+	}
+	return C.GoBytes(unsafe.Pointer(ptr), C.int(length+1))[1:], nil
+}
+
+// NewBinaryTransactionWithData creates a new [pam.BinaryTransaction] from bytes.
+func NewBinaryTransactionWithData(data []byte, retData []byte) BinaryTransaction {
+	t := BinaryTransaction{ReturnedData: retData}
+	t.data = TestBinaryDataEncoder(data)
+	t.ExpectedNull = data == nil
+	return t
+}
+
+// NewBinaryTransactionWithRandomData creates a new [pam.BinaryTransaction] with random data.
+func NewBinaryTransactionWithRandomData(size uint8, retData []byte) BinaryTransaction {
+	t := BinaryTransaction{ReturnedData: retData}
+	randomData := make([]byte, size)
+	if err := binary.Read(rand.Reader, binary.LittleEndian, &randomData); err != nil {
+		panic(err)
+	}
+
+	t.data = TestBinaryDataEncoder(randomData)
+	return t
+}
+
+// Data returns the bytes of the transaction.
+func (b BinaryTransaction) Data() []byte {
+	return b.data
+}
+
+// RespondPAM (not) handles the PAM string conversations.
+func (b BinaryTransaction) RespondPAM(s pam.Style, msg string) (string, error) {
+	return "", errors.Join(pam.ErrConv,
+		&SerializableError{"unexpected non-binary request"})
+}
+
+// RespondPAMBinary handles the PAM binary conversations.
+func (b BinaryTransaction) RespondPAMBinary(ptr pam.BinaryPointer) ([]byte, error) {
+	if ptr == nil && !b.ExpectedNull {
+		return nil, errors.Join(pam.ErrConv,
+			&SerializableError{"unexpected null binary data"})
+	} else if ptr == nil {
+		return TestBinaryDataEncoder(b.ReturnedData), nil
+	}
+
+	bytes, _ := TestBinaryDataDecoder(ptr)
+	if !reflect.DeepEqual(bytes, b.data[1:]) {
+		return nil, errors.Join(pam.ErrConv,
+			&SerializableError{
+				fmt.Sprintf("data mismatch %#v vs %#v", bytes, b.data[1:]),
+			})
+	}
+
+	return TestBinaryDataEncoder(b.ReturnedData), nil
+}

module-transaction-mock.go

@@ -15,6 +15,7 @@ import "C"
 import (
 	"errors"
 	"fmt"
+	"reflect"
 	"runtime"
 	"runtime/cgo"
 	"testing"
@@ -40,10 +41,12 @@ type mockModuleTransaction struct {
 	ConversationHandler ConversationHandler
 	moduleData          map[string]uintptr
 	allocatedData       []unsafe.Pointer
+	binaryProtocol      bool
 }
 
 func newMockModuleTransaction(m *mockModuleTransaction) *mockModuleTransaction {
 	m.moduleData = make(map[string]uintptr)
+	m.binaryProtocol = true
 	runtime.SetFinalizer(m, func(m *mockModuleTransaction) {
 		for _, ptr := range m.allocatedData {
 			C.free(ptr)
@@ -126,14 +129,21 @@ func (m *mockModuleTransaction) getConv() (*C.struct_pam_conv, error) {
 	return nil, nil
 }
 
+func (m *mockModuleTransaction) hasBinaryProtocol() bool {
+	return m.binaryProtocol
+}
+
 type mockConversationHandler struct {
 	User                    string
 	PromptEchoOn            string
 	PromptEchoOff           string
 	TextInfo                string
 	ErrorMsg                string
+	Binary                  []byte
 	ExpectedMessage         string
 	ExpectedMessagesByStyle map[Style]string
+	ExpectedNil             bool
+	ExpectedBinary          []byte
 	CheckEmptyMessage       bool
 	ExpectedStyle           Style
 	CheckZeroStyle          bool
@@ -178,3 +188,46 @@ func (c mockConversationHandler) RespondPAM(s Style, msg string) (string, error)
 
 	return "", fmt.Errorf("%w: unhandled style: %v", ErrConv, s)
 }
+
+func testBinaryDataEncoder(bytes []byte) []byte {
+	if len(bytes) > 0xff {
+		panic("Binary transaction size not supported")
+	}
+
+	if bytes == nil {
+		return bytes
+	}
+
+	data := make([]byte, 0, len(bytes)+1)
+	data = append(data, byte(len(bytes)))
+	data = append(data, bytes...)
+	return data
+}
+
+func testBinaryDataDecoder(ptr BinaryPointer) ([]byte, error) {
+	if ptr == nil {
+		return nil, nil
+	}
+
+	length := uint8(*((*C.uint8_t)(ptr)))
+	if length == 0 {
+		return []byte{}, nil
+	}
+	return C.GoBytes(unsafe.Pointer(ptr), C.int(length+1))[1:], nil
+}
+
+func (c mockConversationHandler) RespondPAMBinary(ptr BinaryPointer) ([]byte, error) {
+	if ptr == nil && !c.ExpectedNil {
+		return nil, fmt.Errorf("%w: unexpected null binary data", ErrConv)
+	} else if ptr == nil {
+		return testBinaryDataEncoder(c.Binary), nil
+	}
+
+	bytes, _ := testBinaryDataDecoder(ptr)
+	if !reflect.DeepEqual(bytes, c.ExpectedBinary) {
+		return nil, fmt.Errorf("%w: data mismatch %#v vs %#v",
+			ErrConv, bytes, c.ExpectedBinary)
+	}
+
+	return testBinaryDataEncoder(c.Binary), nil
+}

module-transaction.go

@@ -9,7 +9,10 @@ import "C"
 import (
 	"errors"
 	"fmt"
+	"runtime"
 	"runtime/cgo"
+	"sync"
+	"sync/atomic"
 	"unsafe"
 )
 
@@ -29,6 +32,7 @@ type ModuleTransaction interface {
 	StartStringConv(style Style, prompt string) (StringConvResponse, error)
 	StartStringConvf(style Style, format string, args ...interface{}) (
 		StringConvResponse, error)
+	StartBinaryConv([]byte) (BinaryConvResponse, error)
 	StartConv(ConvRequest) (ConvResponse, error)
 	StartConvMulti([]ConvRequest) ([]ConvResponse, error)
 }
@@ -110,6 +114,7 @@ type moduleTransactionIface interface {
 	setData(key *C.char, handle C.uintptr_t) C.int
 	getData(key *C.char, outHandle *C.uintptr_t) C.int
 	getConv() (*C.struct_pam_conv, error)
+	hasBinaryProtocol() bool
 	startConv(conv *C.struct_pam_conv, nMsg C.int,
 		messages **C.struct_pam_message,
 		outResponses **C.struct_pam_response) C.int
@@ -261,6 +266,143 @@ func (s stringConvResponse) Response() string {
 	return s.response
 }
 
+// BinaryFinalizer is a type of function that can be used to release
+// the binary when it's not required anymore
+type BinaryFinalizer func(BinaryPointer)
+
+// BinaryConvRequester is the interface that binary ConvRequests should
+// implement
+type BinaryConvRequester interface {
+	ConvRequest
+	Pointer() BinaryPointer
+	CreateResponse(BinaryPointer) BinaryConvResponse
+	Release()
+}
+
+// BinaryConvRequest is a ConvRequest for performing binary conversations.
+type BinaryConvRequest struct {
+	ptr               atomic.Uintptr
+	finalizer         BinaryFinalizer
+	responseFinalizer BinaryFinalizer
+}
+
+// NewBinaryConvRequestFull creates a new BinaryConvRequest with finalizer
+// for response BinaryResponse.
+func NewBinaryConvRequestFull(ptr BinaryPointer, finalizer BinaryFinalizer,
+	responseFinalizer BinaryFinalizer) *BinaryConvRequest {
+	b := &BinaryConvRequest{finalizer: finalizer, responseFinalizer: responseFinalizer}
+	b.ptr.Store(uintptr(ptr))
+	if ptr == nil || finalizer == nil {
+		return b
+	}
+
+	// The ownership of the data here is temporary
+	runtime.SetFinalizer(b, func(b *BinaryConvRequest) { b.Release() })
+	return b
+}
+
+// NewBinaryConvRequest creates a new BinaryConvRequest
+func NewBinaryConvRequest(ptr BinaryPointer, finalizer BinaryFinalizer) *BinaryConvRequest {
+	return NewBinaryConvRequestFull(ptr, finalizer, finalizer)
+}
+
+// NewBinaryConvRequestFromBytes creates a new BinaryConvRequest from an array
+// of bytes.
+func NewBinaryConvRequestFromBytes(bytes []byte) *BinaryConvRequest {
+	if bytes == nil {
+		return &BinaryConvRequest{}
+	}
+	return NewBinaryConvRequest(BinaryPointer(C.CBytes(bytes)),
+		func(ptr BinaryPointer) { C.free(unsafe.Pointer(ptr)) })
+}
+
+// Style returns the response style for the request, so always BinaryPrompt.
+func (b *BinaryConvRequest) Style() Style {
+	return BinaryPrompt
+}
+
+// Pointer returns the conversation style of the StringConvRequest.
+func (b *BinaryConvRequest) Pointer() BinaryPointer {
+	ptr := b.ptr.Load()
+	return *(*BinaryPointer)(unsafe.Pointer(&ptr))
+}
+
+// CreateResponse creates a new BinaryConvResponse from the request
+func (b *BinaryConvRequest) CreateResponse(ptr BinaryPointer) BinaryConvResponse {
+	bcr := &binaryConvResponse{ptr, b.responseFinalizer, &sync.Mutex{}}
+	runtime.SetFinalizer(bcr, func(bcr *binaryConvResponse) {
+		bcr.Release()
+	})
+	return bcr
+}
+
+// Release releases the resources allocated by the request
+func (b *BinaryConvRequest) Release() {
+	ptr := b.ptr.Swap(0)
+	if b.finalizer != nil {
+		b.finalizer(*(*BinaryPointer)(unsafe.Pointer(&ptr)))
+		runtime.SetFinalizer(b, nil)
+	}
+}
+
+// BinaryDecoder is a function type for decode the a binary pointer data into
+// bytes
+type BinaryDecoder func(BinaryPointer) ([]byte, error)
+
+// BinaryConvResponse is a subtype of ConvResponse used for binary
+// conversation responses.
+type BinaryConvResponse interface {
+	ConvResponse
+	Data() BinaryPointer
+	Decode(BinaryDecoder) ([]byte, error)
+	Release()
+}
+
+type binaryConvResponse struct {
+	ptr       BinaryPointer
+	finalizer BinaryFinalizer
+	mutex     *sync.Mutex
+}
+
+// Style returns the response style for the response, so always BinaryPrompt.
+func (b binaryConvResponse) Style() Style {
+	return BinaryPrompt
+}
+
+// Data returns the response native pointer, it's up to the protocol to parse
+// it accordingly.
+func (b *binaryConvResponse) Data() BinaryPointer {
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+	return b.ptr
+}
+
+// Decode decodes the binary data using the provided decoder function.
+func (b *binaryConvResponse) Decode(decoder BinaryDecoder) (
+	[]byte, error) {
+	if decoder == nil {
+		return nil, errors.New("nil decoder provided")
+	}
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+	return decoder(b.ptr)
+}
+
+// Release releases the binary conversation response data.
+// This is also automatically via a finalizer, but applications may control
+// this explicitly deferring execution of this.
+func (b *binaryConvResponse) Release() {
+	b.mutex.Lock()
+	defer b.mutex.Unlock()
+	ptr := b.ptr
+	b.ptr = nil
+	if b.finalizer != nil {
+		b.finalizer(ptr)
+	} else {
+		C.free(unsafe.Pointer(ptr))
+	}
+}
+
 // StartStringConv starts a text-based conversation using the provided style
 // and prompt.
 func (m *moduleTransaction) StartStringConv(style Style, prompt string) (
@@ -291,6 +433,29 @@ func (m *moduleTransaction) StartStringConvf(style Style, format string, args ..
 	return m.StartStringConv(style, fmt.Sprintf(format, args...))
 }
 
+// HasBinaryProtocol checks if binary protocol is supported.
+func (m *moduleTransaction) hasBinaryProtocol() bool {
+	return CheckPamHasBinaryProtocol()
+}
+
+// StartBinaryConv starts a binary conversation using the provided bytes.
+func (m *moduleTransaction) StartBinaryConv(bytes []byte) (
+	BinaryConvResponse, error) {
+	return m.startBinaryConvImpl(m, bytes)
+}
+
+func (m *moduleTransaction) startBinaryConvImpl(iface moduleTransactionIface,
+	bytes []byte) (
+	BinaryConvResponse, error) {
+	res, err := m.startConvImpl(iface, NewBinaryConvRequestFromBytes(bytes))
+	if err != nil {
+		return nil, err
+	}
+
+	binaryRes, _ := res.(BinaryConvResponse)
+	return binaryRes, nil
+}
+
 // StartConv initiates a PAM conversation using the provided ConvRequest.
 func (m *moduleTransaction) StartConv(req ConvRequest) (
 	ConvResponse, error) {
@@ -360,14 +525,21 @@ func (m *moduleTransaction) startConvMultiImpl(iface moduleTransactionIface,
 		case StringConvRequest:
 			cBytes = unsafe.Pointer(C.CString(r.Prompt()))
 			defer C.free(cBytes)
+		case BinaryConvRequester:
+			if !iface.hasBinaryProtocol() {
+				return nil, errors.New("%w: binary protocol is not supported")
+			}
+			cBytes = unsafe.Pointer(r.Pointer())
 		default:
 			return nil, fmt.Errorf("unsupported conversation type %#v", r)
 		}
 
-		goMsgs[i] = &C.struct_pam_message{
-			msg_style: C.int(req.Style()),
-			msg:       (*C.char)(cBytes),
-		}
+		cMessage := (*C.struct_pam_message)(C.calloc(1,
+			(C.size_t)(unsafe.Sizeof(*goMsgs[i]))))
+		defer C.free(unsafe.Pointer(cMessage))
+		cMessage.msg_style = C.int(req.Style())
+		cMessage.msg = (*C.char)(cBytes)
+		goMsgs[i] = cMessage
 	}
 
 	var cResponses *C.struct_pam_response
@@ -378,15 +550,26 @@ func (m *moduleTransaction) startConvMultiImpl(iface moduleTransactionIface,
 
 	goResponses := unsafe.Slice(cResponses, len(requests))
 	defer func() {
-		for _, resp := range goResponses {
-			C.free(unsafe.Pointer(resp.resp))
+		for i, resp := range goResponses {
+			if resp.resp == nil {
+				continue
+			}
+			switch req := requests[i].(type) {
+			case BinaryConvRequester:
+				// In the binary prompt case, we need to rely on the provided
+				// finalizer to release the response, so let's create a new one.
+				req.CreateResponse(BinaryPointer(resp.resp)).Release()
+			default:
+				C.free(unsafe.Pointer(resp.resp))
+			}
 		}
 		C.free(unsafe.Pointer(cResponses))
 	}()
 
 	responses = make([]ConvResponse, 0, len(requests))
 	for i, resp := range goResponses {
-		msgStyle := requests[i].Style()
+		request := requests[i]
+		msgStyle := request.Style()
 		switch msgStyle {
 		case PromptEchoOff:
 			fallthrough
@@ -399,6 +582,13 @@ func (m *moduleTransaction) startConvMultiImpl(iface moduleTransactionIface,
 				style:    msgStyle,
 				response: C.GoString(resp.resp),
 			})
+		case BinaryPrompt:
+			// Let's steal the resp ownership here, so that the request
+			// finalizer won't act on it.
+			bcr, _ := request.(BinaryConvRequester)
+			resp := bcr.CreateResponse(BinaryPointer(resp.resp))
+			goResponses[i].resp = nil
+			responses = append(responses, resp)
 		default:
 			return nil,
 				fmt.Errorf("unsupported conversation type %v", msgStyle)

module-transaction_test.go

@@ -123,6 +123,11 @@ func Test_NewNullModuleTransaction(t *testing.T) {
 				return mt.StartConvMulti([]ConvRequest{
 					NewStringConvRequest(TextInfo, "a prompt"),
 					NewStringConvRequest(ErrorMsg, "another prompt"),
+					NewBinaryConvRequest(BinaryPointer(&mt), nil),
+					NewBinaryConvRequestFromBytes([]byte("These are bytes!")),
+					NewBinaryConvRequestFromBytes([]byte{}),
+					NewBinaryConvRequestFromBytes(nil),
+					NewBinaryConvRequest(nil, nil),
 				})
 			},
 		},
@@ -620,31 +625,272 @@ func Test_MockModuleTransaction(t *testing.T) {
 			},
 		},
 		"StartConvMulti-all-types": {
-			expectedValue: []ConvResponse{
-				stringConvResponse{TextInfo, "nice to see you, Go!"},
-				stringConvResponse{ErrorMsg, "ops, sorry..."},
-				stringConvResponse{PromptEchoOn, "here's my public data"},
-				stringConvResponse{PromptEchoOff, "here's my private data"},
+			expectedValue: []any{
+				[]ConvResponse{
+					stringConvResponse{TextInfo, "nice to see you, Go!"},
+					stringConvResponse{ErrorMsg, "ops, sorry..."},
+					stringConvResponse{PromptEchoOn, "here's my public data"},
+					stringConvResponse{PromptEchoOff, "here's my private data"},
+				},
+				[][]byte{
+					{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+				},
 			},
 			conversationHandler: mockConversationHandler{
 				TextInfo:      "nice to see you, Go!",
 				ErrorMsg:      "ops, sorry...",
 				PromptEchoOn:  "here's my public data",
 				PromptEchoOff: "here's my private data",
+				Binary:        []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
 				ExpectedMessagesByStyle: map[Style]string{
 					TextInfo:      "hello PAM!",
 					ErrorMsg:      "This is wrong, PAM!",
 					PromptEchoOn:  "Give me your non-private infos",
 					PromptEchoOff: "Give me your private secrets",
 				},
+				ExpectedBinary: []byte("\x00This is a binary data request\xC5\x00\xffYes it is!"),
 			},
 			testFunc: func(mock *mockModuleTransaction) (any, error) {
-				return mt.startConvMultiImpl(mock, []ConvRequest{
+				requests := []ConvRequest{
 					NewStringConvRequest(TextInfo, "hello PAM!"),
 					NewStringConvRequest(ErrorMsg, "This is wrong, PAM!"),
 					NewStringConvRequest(PromptEchoOn, "Give me your non-private infos"),
 					NewStringConvRequest(PromptEchoOff, "Give me your private secrets"),
-				})
+					NewBinaryConvRequestFromBytes(
+						testBinaryDataEncoder([]byte("\x00This is a binary data request\xC5\x00\xffYes it is!"))),
+				}
+
+				data, err := mt.startConvMultiImpl(mock, requests)
+				if err != nil {
+					return data, err
+				}
+
+				stringResponses := []ConvResponse{}
+				binaryResponses := [][]byte{}
+				for i, r := range data {
+					if r.Style() != requests[i].Style() {
+						mock.T.Fatalf("unexpected style %#v vs %#v",
+							r.Style(), requests[i].Style())
+					}
+
+					switch rt := r.(type) {
+					case BinaryConvResponse:
+						decoded, err := rt.Decode(testBinaryDataDecoder)
+						if err != nil {
+							return data, err
+						}
+						binaryResponses = append(binaryResponses, decoded)
+					case StringConvResponse:
+						stringResponses = append(stringResponses, r)
+					default:
+						mock.T.Fatalf("unexpected value %v", rt)
+					}
+				}
+				return []any{
+					stringResponses,
+					binaryResponses,
+				}, err
+			},
+		},
+		"StartConvMulti-all-types-some-failing": {
+			expectedError: ErrConv,
+			expectedValue: []ConvResponse(nil),
+			conversationHandler: mockConversationHandler{
+				TextInfo:      "nice to see you, Go!",
+				ErrorMsg:      "ops, sorry...",
+				PromptEchoOn:  "here's my public data",
+				PromptEchoOff: "here's my private data",
+				Binary:        []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+				ExpectedMessagesByStyle: map[Style]string{
+					TextInfo:      "hello PAM!",
+					ErrorMsg:      "This is wrong, PAM!",
+					PromptEchoOn:  "Give me your non-private infos",
+					PromptEchoOff: "Give me your private secrets",
+					Style(0xfaaf): "This will fail",
+				},
+				ExpectedBinary:     []byte("\x00This is a binary data request\xC5\x00\xffYes it is!"),
+				IgnoreUnknownStyle: true,
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				requests := []ConvRequest{
+					NewStringConvRequest(TextInfo, "hello PAM!"),
+					NewStringConvRequest(ErrorMsg, "This is wrong, PAM!"),
+					NewStringConvRequest(PromptEchoOn, "Give me your non-private infos"),
+					NewStringConvRequest(PromptEchoOff, "Give me your private secrets"),
+					NewStringConvRequest(Style(0xfaaf), "This will fail"),
+					NewBinaryConvRequestFromBytes(
+						testBinaryDataEncoder([]byte("\x00This is a binary data request\xC5\x00\xffYes it is!"))),
+				}
+
+				return mt.startConvMultiImpl(mock, requests)
+			},
+		},
+		"StartConv-Binary-unsupported": {
+			expectedValue: nil,
+			expectedError: ErrConv,
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x00This is a binary data request\xC5\x00\xffYes it is!"),
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				mock.binaryProtocol = false
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is!"))
+				return mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(bytes))
+			},
+		},
+		"StartConv-Binary": {
+			expectedValue: []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x00This is a binary data request\xC5\x00\xffYes it is!"),
+				Binary:         []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is!"))
+				data, err := mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(bytes))
+				if err != nil {
+					return data, err
+				}
+				bcr, _ := data.(BinaryConvResponse)
+				return bcr.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartConv-Binary-expected-data-mismatch": {
+			expectedError: ErrConv,
+			expectedValue: nil,
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x00This is not the expected data!"),
+				Binary:         []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is!"))
+				return mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(bytes))
+			},
+		},
+		"StartConv-Binary-unexpected-nil": {
+			expectedError: ErrConv,
+			expectedValue: nil,
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x00This should not be nil"),
+				Binary:         []byte("\x1ASome binary Dat\xaa"),
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				return mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(nil))
+			},
+		},
+		"StartConv-Binary-expected-nil": {
+			expectedValue: []byte("\x1ASome binary Dat\xaa"),
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedNil:    true,
+				ExpectedBinary: []byte("\x00This should not be nil"),
+				Binary:         []byte("\x1ASome binary Dat\xaa"),
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				data, err := mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(nil))
+				if err != nil {
+					return data, err
+				}
+				bcr, _ := data.(BinaryConvResponse)
+				return bcr.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartConv-Binary-returns-nil": {
+			expectedValue: BinaryPointer(nil),
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x1ASome binary Dat\xaa"),
+				Binary:         nil,
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte("\x1ASome binary Dat\xaa"))
+				data, err := mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(bytes))
+				if err != nil {
+					return data, err
+				}
+				bcr, _ := data.(BinaryConvResponse)
+				return bcr.Data(), err
+			},
+		},
+		"StartBinaryConv": {
+			expectedValue: []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x00This is a binary data request\xC5\x00\xffYes it is!"),
+				Binary:         []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is!"))
+				data, err := mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(bytes))
+				if err != nil {
+					return data, err
+				}
+				bcr, _ := data.(BinaryConvResponse)
+				return bcr.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartBinaryConv-expected-data-mismatch": {
+			expectedError: ErrConv,
+			expectedValue: nil,
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x00This is not the expected data!"),
+				Binary:         []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is!"))
+				return mt.startBinaryConvImpl(mock, bytes)
+			},
+		},
+		"StartBinaryConv-unexpected-nil": {
+			expectedError: ErrConv,
+			expectedValue: nil,
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x00This should not be nil"),
+				Binary:         []byte("\x1ASome binary Dat\xaa"),
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				return mt.startBinaryConvImpl(mock, nil)
+			},
+		},
+		"StartBinaryConv-expected-nil": {
+			expectedValue: []byte("\x1ASome binary Dat\xaa"),
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedNil:    true,
+				ExpectedBinary: []byte("\x00This should not be nil"),
+				Binary:         []byte("\x1ASome binary Dat\xaa"),
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				data, err := mt.startBinaryConvImpl(mock, nil)
+				if err != nil {
+					return data, err
+				}
+				return data.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartBinaryConv-returns-nil": {
+			expectedValue: BinaryPointer(nil),
+			conversationHandler: mockConversationHandler{
+				ExpectedStyle:  BinaryPrompt,
+				ExpectedBinary: []byte("\x1ASome binary Dat\xaa"),
+				Binary:         nil,
+			},
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte("\x1ASome binary Dat\xaa"))
+				data, err := mt.startBinaryConvImpl(mock, bytes)
+				if err != nil {
+					return data, err
+				}
+				return data.Data(), err
 			},
 		},
 	}

transaction.h

@@ -42,7 +42,10 @@ static inline int cb_pam_conv(int num_msg, PAM_CONST struct pam_message **msg, s
 error:
 	for (size_t i = 0; i < num_msg; ++i) {
 		if ((*resp)[i].resp) {
-			memset((*resp)[i].resp, 0, strlen((*resp)[i].resp));
+#ifdef PAM_BINARY_PROMPT
+			if (msg[i]->msg_style != PAM_BINARY_PROMPT)
+#endif
+				memset((*resp)[i].resp, 0, strlen((*resp)[i].resp));
 			free((*resp)[i].resp);
 		}
 	}