Decode text in the database encoding in SqliteDataReader

src/Microsoft.Data.Sqlite.Core/SqliteDataRecord.cs

@@ -35,6 +35,7 @@ internal class RowIdInfo(int ordinal, string tableName)
 
     private bool _alreadyThrown;
     private bool _alreadyAddedChanges;
+    private bool? _databaseEncodingIsUtf8;
 
     public virtual object this[string name]
         => GetValue(GetOrdinal(name));
@@ -266,7 +267,7 @@ public virtual long GetBytes(int ordinal, long dataOffset, byte[]? buffer, int b
 
     public virtual long GetChars(int ordinal, long dataOffset, char[]? buffer, int bufferOffset, int length)
     {
-        using var reader = new StreamReader(GetStream(ordinal), Encoding.UTF8);
+        using var reader = CreateTextReader(ordinal);
 
         if (buffer == null)
         {
@@ -400,8 +401,17 @@ public virtual Stream GetStream(int ordinal)
     public virtual TextReader GetTextReader(int ordinal)
         => IsDBNull(ordinal)
             ? new StringReader(string.Empty)
+            : CreateTextReader(ordinal);
+
+    private TextReader CreateTextReader(int ordinal)
+        // A stream exposes text in the database encoding. SQLite only converts it when it's read as a string.
+        => GetSqliteType(ordinal) == SQLITE_TEXT && !DatabaseEncodingIsUtf8()
+            ? new StringReader(GetString(ordinal))
             : new StreamReader(GetStream(ordinal), Encoding.UTF8);
 
+    private bool DatabaseEncodingIsUtf8()
+        => _databaseEncodingIsUtf8 ??= connection.ExecuteScalar<string>("PRAGMA encoding;") == "UTF-8";
+
     public bool Read()
     {
         if (!_stepped)

test/Microsoft.Data.Sqlite.Tests/SqliteDataReaderTest.cs

@@ -279,6 +279,32 @@ public void GetChars_works_with_overflow()
         }
     }
 
+    [Theory,
+     InlineData("UTF-16le", "SELECT rowid, Value FROM Data;"),
+     InlineData("UTF-16be", "SELECT rowid, Value FROM Data;"),
+     InlineData("UTF-16le", "SELECT 0, Value FROM Data;"),
+     InlineData("UTF-16be", "SELECT 0, Value FROM Data;"),
+     InlineData("UTF-16le", "SELECT 0, Value || '' FROM Data;"),
+     InlineData("UTF-8", "SELECT rowid, Value FROM Data;")]
+    public void GetChars_works_for_database_encoding(string encoding, string query)
+    {
+        using var connection = new SqliteConnection("Data Source=:memory:");
+        connection.Open();
+
+        connection.ExecuteNonQuery(
+            $"PRAGMA encoding = '{encoding}'; CREATE TABLE Data (Value TEXT); INSERT INTO Data VALUES ('têst');");
+
+        using var reader = connection.ExecuteReader(query);
+        var hasData = reader.Read();
+        Assert.True(hasData);
+
+        var buffer = new char[2];
+        var charsRead = reader.GetChars(1, 1, buffer, 0, buffer.Length);
+        Assert.Equal(2, charsRead);
+        Assert.Equal(new[] { 'ê', 's' }, buffer);
+        Assert.Equal(4, reader.GetChars(1, 0, null, 0, 0));
+    }
+
     [Fact]
     public void GetChars_throws_when_dataOffset_out_of_range()
     {
@@ -616,6 +642,83 @@ public void GetTextReader_works()
         Assert.Equal("test", textReader.ReadToEnd());
     }
 
+    [Theory,
+     InlineData("UTF-16le", "SELECT rowid, Value FROM Data;"),
+     InlineData("UTF-16be", "SELECT rowid, Value FROM Data;"),
+     InlineData("UTF-16le", "SELECT 0, Value FROM Data;"),
+     InlineData("UTF-16be", "SELECT 0, Value FROM Data;"),
+     InlineData("UTF-16le", "SELECT 0, Value || '' FROM Data;"),
+     InlineData("UTF-8", "SELECT rowid, Value FROM Data;")]
+    public void GetTextReader_works_for_database_encoding(string encoding, string query)
+    {
+        using var connection = new SqliteConnection("Data Source=:memory:");
+        connection.Open();
+
+        connection.ExecuteNonQuery(
+            $"PRAGMA encoding = '{encoding}'; CREATE TABLE Data (Value TEXT); INSERT INTO Data VALUES ('têst');");
+
+        using var reader = connection.ExecuteReader(query);
+        var hasData = reader.Read();
+        Assert.True(hasData);
+
+        using var textReader = reader.GetTextReader(1);
+        Assert.Equal("têst", textReader.ReadToEnd());
+    }
+
+    [Fact]
+    public void GetTextReader_works_for_parameter_when_UTF16_database()
+    {
+        using var connection = new SqliteConnection("Data Source=:memory:");
+        connection.Open();
+
+        connection.ExecuteNonQuery("PRAGMA encoding = 'UTF-16le'; CREATE TABLE Data (Value TEXT);");
+
+        using var command = connection.CreateCommand();
+        command.CommandText = "SELECT $value;";
+        command.Parameters.AddWithValue("$value", "têst");
+
+        using var reader = command.ExecuteReader();
+        var hasData = reader.Read();
+        Assert.True(hasData);
+
+        using var textReader = reader.GetTextReader(0);
+        Assert.Equal("têst", textReader.ReadToEnd());
+    }
+
+    [Fact]
+    public void GetTextReader_reads_blob_as_UTF8_when_UTF16_database()
+    {
+        using var connection = new SqliteConnection("Data Source=:memory:");
+        connection.Open();
+
+        connection.ExecuteNonQuery("PRAGMA encoding = 'UTF-16le'; CREATE TABLE Data (Value BLOB);");
+        connection.ExecuteNonQuery("INSERT INTO Data VALUES (X'74C3AA7374');");
+
+        using var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;");
+        var hasData = reader.Read();
+        Assert.True(hasData);
+
+        using var textReader = reader.GetTextReader(1);
+        Assert.Equal("têst", textReader.ReadToEnd());
+    }
+
+    [Fact]
+    public void GetFieldValue_of_TextReader_works_when_UTF16_database()
+    {
+        using var connection = new SqliteConnection("Data Source=:memory:");
+        connection.Open();
+
+        connection.ExecuteNonQuery(
+            "PRAGMA encoding = 'UTF-16le'; CREATE TABLE Data (Value TEXT); INSERT INTO Data VALUES ('têst');");
+
+        using var reader = connection.ExecuteReader("SELECT rowid, Value FROM Data;");
+        var hasData = reader.Read();
+        Assert.True(hasData);
+
+        using var textReader = reader.GetFieldValue<TextReader>(1);
+        Assert.Equal("têst", textReader.ReadToEnd());
+    }
+
     [Fact]
     public void GetTextReader_works_when_null()
     {