[GR-74749] Treat foreign host buffers as Python buffers.

PullRequest: graalpython/4374

Author: timfel

CHANGELOG.md

@@ -4,6 +4,7 @@ This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
 ## Version 25.1.0
+* Treat foreign buffer objects as Python buffer-compatible binary objects, so APIs like `memoryview`, `bytes`, `bytearray`, `binascii.hexlify`, and `io.BytesIO` work naturally on them when embedding GraalPy in Java. This allows passing binary data between Python and Java's `ByteBuffer` and `ByteSequence` types with minimal (sometimes zero) copies.
 * Add support for [Truffle source options](https://www.graalvm.org/truffle/javadoc/com/oracle/truffle/api/source/Source.SourceBuilder.html#option(java.lang.String,java.lang.String)):
   * The `python.Optimize` option can be used to specify the optimization level, like the `-O` (level 1) and `-OO` (level 2) commandline options.
   * The `python.NewGlobals` option can be used to run a source with a fresh globals dictionary instead of the main module globals, which is useful for embeddings that want isolated top-level execution.

docs/user/Interoperability.md

@@ -118,6 +118,60 @@ assert l == [6]
 
 See the [Interop Types to Python](#interop-types-to-python) section for more interop traits and how they map to Python types.
 
+## Passing Binary Data Between Java and Python
+
+Passing binary data between Java and Python deserves attention:
+
+- Java code typically uses `byte[]` or `java.nio.ByteBuffer`
+- Python code typically uses `bytes`, `bytearray`, `memoryview`, or file-like APIs such as `io.BytesIO`
+
+### Java to Python
+
+Raw Java `byte[]` are accessible as `list`-like objects in Python.
+Only integral values that fit into a signed `byte` can be read from or written to such objects.
+Python, on the other hand, usually exposes binary data as unsigned byte values.
+To achieve the equivalent of a "re-interpreting cast", Java byte arrays should be passed to Python using `ByteBuffer.wrap(byte[])`:
+
+```java
+import java.nio.ByteBuffer;
+byte[] data = ...;
+ByteBuffer buffer = ByteBuffer.wrap(data); // does not copy
+context.getBindings("python").putMember("java_buffer", buffer);
+```
+
+Python can then use the object through buffer-oriented binary data APIs:
+
+```python
+memoryview(java_buffer)  # does not copy
+bytes(java_buffer)       # copies into an immutable Python-owned buffer
+bytearray(java_buffer)   # copies into a mutable Python-owned buffer
+io.BytesIO(java_buffer)  # copies into BytesIO's internal storage
+```
+
+### Python to Java
+
+Python `bytes` and other bytes-like objects can be interpreted like any `java.lang.List`.
+Because Python bytes are usually unsigned, however, they cannot simply be converted via `Value#as(byte[].class)` if any values are larger than 127.
+The Graal polyglot sdk provides `org.graalvm.polyglot.io.ByteSequence` as a target type to deal with this issue explicitly.
+
+```java
+import org.graalvm.polyglot.Value;
+import org.graalvm.polyglot.io.ByteSequence;
+Value result = context.eval("python", "b'hello'");
+ByteSequence seq = result.as(ByteSequence.class);   // does not copy
+```
+
+`ByteSequence` keeps the data as a Python-owned byte sequence without immediately copying.
+It provides a `toByteArray()` method that deals with re-interpreting unsigned Python bytes as signed Java bytes.
+
+```java
+import java.nio.charset.StandardCharsets;
+import org.graalvm.polyglot.io.ByteSequence;
+ByteSequence seq = result.as(ByteSequence.class);
+byte[] bytes = seq.toByteArray();  // copies into Java byte[]
+String s = new String(bytes, StandardCharsets.UTF_8);
+```
+
 ## Call Other Languages from Python
 
 The _polyglot_ API allows non-JVM specific interactions with other languages from Python scripts.

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/interop/HostInteropTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -46,13 +46,15 @@
 import static org.junit.Assert.assertTrue;
 
 import java.math.BigInteger;
+import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.time.LocalDate;
 import java.time.LocalTime;
 import java.time.ZoneId;
 
 import org.graalvm.polyglot.Context;
 import org.graalvm.polyglot.Value;
+import org.graalvm.polyglot.io.ByteSequence;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -713,4 +715,79 @@ public void testByteBuffer() {
         t.writeBufferDouble(ByteOrder.LITTLE_ENDIAN, 0, 12345.6789123);
         assertEquals(12345.6789123, t.readBufferDouble(ByteOrder.LITTLE_ENDIAN, 0), 0.0);
     }
+
+    @Test
+    public void testHostByteBufferAsPythonBuffer() {
+        byte[] writable = new byte[]{1, 2, 3, 4};
+        context.getBindings("python").putMember("writable_bb", ByteBuffer.wrap(writable));
+        context.getBindings("python").putMember("readonly_bb", ByteBuffer.wrap(new byte[]{-1, 5, 6, 7, 8}).asReadOnlyBuffer());
+
+        context.eval("python", """
+                        import binascii
+                        import io
+
+                        mv = memoryview(writable_bb)
+                        assert not mv.readonly
+                        assert mv.tobytes() == b"\\x01\\x02\\x03\\x04"
+                        assert bytes(writable_bb) == b"\\x01\\x02\\x03\\x04"
+                        assert bytearray(writable_bb) == bytearray(b"\\x01\\x02\\x03\\x04")
+                        assert binascii.hexlify(writable_bb) == b"01020304"
+                        bio = io.BytesIO()
+                        assert bio.write(writable_bb) == 4
+                        assert bio.getvalue() == b"\\x01\\x02\\x03\\x04"
+                        mv[1] = 9
+                        assert io.BytesIO(b"abcd").readinto(writable_bb) == 4
+                        assert bytes(writable_bb) == b"abcd"
+
+                        ro = memoryview(readonly_bb)
+                        assert ro.readonly
+                        assert ro.tobytes() == b"\\xff\\x05\\x06\\x07\\x08"
+                        assert bytes(readonly_bb) == b"\\xff\\x05\\x06\\x07\\x08"
+                        assert bytearray(readonly_bb) == bytearray(b"\\xff\\x05\\x06\\x07\\x08")
+                        assert io.BytesIO().write(readonly_bb) == 5
+                        try:
+                            ro[0] = 1
+                            raise AssertionError("expected memoryview write to fail")
+                        except TypeError:
+                            pass
+                        try:
+                            io.BytesIO(b"wxyz").readinto(readonly_bb)
+                            raise AssertionError("expected readinto to fail")
+                        except TypeError:
+                            pass
+                        """);
+
+        assertArrayEquals(new byte[]{'a', 'b', 'c', 'd'}, writable);
+    }
+
+    @Test
+    public void testHostByteSequenceAsPythonBuffer() {
+        byte[] bytes = new byte[]{10, 20, 30, 40};
+        context.getBindings("python").putMember("seq", ByteSequence.create(bytes));
+
+        context.eval("python", """
+                        import binascii
+                        import io
+
+                        mv = memoryview(seq)
+                        assert mv.readonly
+                        assert mv.tobytes() == b"\\x0a\\x14\\x1e\\x28"
+                        assert bytes(seq) == b"\\x0a\\x14\\x1e\\x28"
+                        assert bytearray(seq) == bytearray(b"\\x0a\\x14\\x1e\\x28")
+                        assert binascii.hexlify(seq) == b"0a141e28"
+                        bio = io.BytesIO()
+                        assert bio.write(seq) == 4
+                        assert bio.getvalue() == b"\\x0a\\x14\\x1e\\x28"
+                        try:
+                            mv[0] = 1
+                            raise AssertionError("expected memoryview write to fail")
+                        except TypeError:
+                            pass
+                        try:
+                            io.BytesIO(b"abcd").readinto(seq)
+                            raise AssertionError("expected readinto to fail")
+                        except TypeError:
+                            pass
+                        """);
+    }
 }