Map byte[] to json/jsonb as a raw JSON document again

src/EFCore.PG/Query/ExpressionTranslators/Internal/NpgsqlMiscAggregateMethodTranslator.cs

@@ -1,4 +1,5 @@
 using Npgsql.EntityFrameworkCore.PostgreSQL.Query.Expressions.Internal;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Storage.Internal;
 using static Npgsql.EntityFrameworkCore.PostgreSQL.Utilities.Statics;
 
 namespace Npgsql.EntityFrameworkCore.PostgreSQL.Query.ExpressionTranslators.Internal;
@@ -104,7 +105,7 @@ public class NpgsqlMiscAggregateMethodTranslator(
                         nullable: true,
                         argumentsPropagateNullability: FalseArrays[1],
                         returnType: method.ReturnType,
-                        _typeMappingSource.FindMapping(method.ReturnType, "json"));
+                        FindJsonArrayMapping(method.ReturnType, "json"));
 
                 case nameof(NpgsqlAggregateDbFunctionsExtensions.JsonbAgg):
                     return _sqlExpressionFactory.AggregateFunction(
@@ -114,7 +115,7 @@ public class NpgsqlMiscAggregateMethodTranslator(
                         nullable: true,
                         argumentsPropagateNullability: FalseArrays[1],
                         returnType: method.ReturnType,
-                        _typeMappingSource.FindMapping(method.ReturnType, "jsonb"));
+                        FindJsonArrayMapping(method.ReturnType, "jsonb"));
 
                 case nameof(NpgsqlAggregateDbFunctionsExtensions.Sum):
                     return _sqlExpressionFactory.AggregateFunction(
@@ -191,4 +192,15 @@ public class NpgsqlMiscAggregateMethodTranslator(
 
         return null;
     }
+
+    // json_agg/jsonb_agg return a JSON array of the aggregated values, so we explicitly request a JSON collection mapping by passing
+    // the element mapping. Without it, byte[] (from JsonAgg<byte>/JsonbAgg<byte>) would get the opaque mapping used for byte[]
+    // properties mapped to json/jsonb, which reads the raw UTF-8 bytes of the JSON array instead of its elements.
+    // If there's no collection mapping for the element type, we fall back to the regular mapping lookup.
+    private RelationalTypeMapping? FindJsonArrayMapping(Type arrayType, string storeType)
+        => _typeMappingSource.FindMapping(arrayType.GetElementType()!) is { } elementMapping
+            && ((NpgsqlTypeMappingSource)_typeMappingSource).FindCollectionMapping(
+                storeType, arrayType, providerClrType: null, elementMapping) is { } collectionMapping
+                ? collectionMapping
+                : _typeMappingSource.FindMapping(arrayType, storeType);
 }

src/EFCore.PG/Storage/Internal/Mapping/NpgsqlJsonTypeMapping.cs

@@ -108,6 +108,8 @@ protected override string GenerateNonNullSqlLiteral(object value)
             }
             case string s:
                 return $"'{EscapeSqlLiteral(s)}'";
+            case byte[] bytes:
+                return $"'{EscapeSqlLiteral(Encoding.UTF8.GetString(bytes))}'";
             default: // User POCO
                 return $"'{EscapeSqlLiteral(JsonSerializer.Serialize(value))}'";
         }

src/EFCore.PG/Storage/Internal/NpgsqlTypeMappingSource.cs

@@ -745,6 +745,13 @@ public NpgsqlTypeMappingSource(
             // However, when scalar collections are nested inside a JSON document, they obviously have to be JSON-mapped.
             // In addition, at least in theory users may decide to map to JSON instead of array even outside JSON documents.
             case "jsonb" or "json" when modelClrType is not null:
+                // byte[] mapped to json/jsonb represents the raw UTF-8 JSON document (as supported by Npgsql), rather than a primitive
+                // collection of bytes; return null to let the calling code in FindBaseMapping return an NpgsqlJsonTypeMapping.
+                if (modelClrType == typeof(byte[]) && elementMapping is null)
+                {
+                    return null;
+                }
+
                 var mapping = base.FindCollectionMapping(
                     new RelationalTypeMappingInfo(
                         modelClrType, (RelationalTypeMapping?)elementMapping, storeTypeName: storeType, storeTypeNameBase: storeType),

test/EFCore.PG.FunctionalTests/Query/NorthwindFunctionsQueryNpgsqlTest.cs

@@ -701,6 +701,30 @@ GROUP BY c."City"
 """);
     }
 
+    [ConditionalTheory]
+    [InlineData("jsonb")]
+    [InlineData("json")]
+    public virtual async Task GroupBy_Json_aggregate_over_byte_values(string storeType)
+    {
+        await using var context = CreateContext();
+
+        var groups = context.Set<Employee>().Where(e => e.EmployeeID <= 2).GroupBy(e => e.Country);
+        var values = storeType == "jsonb"
+            ? await groups.Select(g => EF.Functions.JsonbAgg(g.Select(e => (byte)e.EmployeeID).OrderBy(v => v))).SingleAsync()
+            : await groups.Select(g => EF.Functions.JsonAgg(g.Select(e => (byte)e.EmployeeID).OrderBy(v => v))).SingleAsync();
+
+        Assert.Equal(new byte[] { 1, 2 }, values);
+
+        AssertSql(
+            $"""
+SELECT {storeType}_agg(e."EmployeeID"::smallint ORDER BY e."EmployeeID"::smallint NULLS FIRST)
+FROM "Employees" AS e
+WHERE e."EmployeeID" <= 2
+GROUP BY e."Country"
+LIMIT 2
+""");
+    }
+
     #endregion Aggregate functions
 
     #region JsonObjectAgg

test/EFCore.PG.FunctionalTests/Query/Translations/JsonTranslationsNpgsqlTest.cs

@@ -1,3 +1,5 @@
+using System.Text.Json;
+
 namespace Microsoft.EntityFrameworkCore.Query.Translations;
 
 public class JsonTranslationsNpgsqlTest : JsonTranslationsRelationalTestBase<JsonTranslationsNpgsqlTest.JsonTranslationsQueryNpgsqlFixture>
@@ -48,6 +50,100 @@ WHERE JSON_EXISTS(j."JsonOwnedType", '$.OptionalInt')
 """);
     }
 
+    [ConditionalTheory]
+    [InlineData("jsonb")]
+    [InlineData("json")]
+    public virtual async Task Byte_array_mapped_to_json_column(string storeType)
+    {
+        await using var context = CreateContext();
+        await context.Database.CreateExecutionStrategy().ExecuteAsync(
+            async () =>
+            {
+                await using var transaction = await context.Database.BeginTransactionAsync();
+
+                context.Add(
+                    new ByteArrayJsonEntity
+                    {
+                        Id = 1,
+                        Json = """{"Foo":"Bar"}"""u8.ToArray(),
+                        Jsonb = """{"Foo":"Bar"}"""u8.ToArray()
+                    });
+                await context.SaveChangesAsync();
+
+                // Simulate a JSON document which wasn't written via EF.
+                await context.Database.ExecuteSqlRawAsync(
+                    """
+INSERT INTO "ByteArrayJsonEntities" ("Id", "Json", "Jsonb") VALUES (2, '{{"Foo":"Baz"}}', '{{"Foo":"Baz"}}')
+""");
+
+                var column = storeType == "jsonb" ? nameof(ByteArrayJsonEntity.Jsonb) : nameof(ByteArrayJsonEntity.Json);
+                var sql = $$"""SELECT "{{column}}" ->> 'Foo' AS "Value" FROM "ByteArrayJsonEntities" WHERE "Id" = {0}""";
+
+                Assert.Equal("Bar", await context.Database.SqlQueryRaw<string>(sql, 1).SingleAsync());
+
+                var entities = await context.Set<ByteArrayJsonEntity>().AsNoTracking().OrderBy(e => e.Id).ToListAsync();
+                Assert.Equal(2, entities.Count);
+                using var efDocument = JsonDocument.Parse(storeType == "jsonb" ? entities[0].Jsonb : entities[0].Json);
+                using var externalDocument = JsonDocument.Parse(storeType == "jsonb" ? entities[1].Jsonb : entities[1].Json);
+                Assert.Equal("Bar", efDocument.RootElement.GetProperty("Foo").GetString());
+                Assert.Equal("Baz", externalDocument.RootElement.GetProperty("Foo").GetString());
+
+                context.Attach(entities[1]);
+                entities[1].Json = """{"Foo":"Qux"}"""u8.ToArray();
+                entities[1].Jsonb = """{"Foo":"Qux"}"""u8.ToArray();
+                await context.SaveChangesAsync();
+
+                Assert.Equal("Qux", await context.Database.SqlQueryRaw<string>(sql, 2).SingleAsync());
+            });
+    }
+
+    [ConditionalTheory, MemberData(nameof(IsAsyncData))]
+    public virtual async Task Byte_array_mapped_to_json_column_in_predicate(bool async)
+    {
+        await using var context = CreateContext();
+        await context.Database.CreateExecutionStrategy().ExecuteAsync(
+            async () =>
+            {
+                await using var transaction = await context.Database.BeginTransactionAsync();
+
+                var json = """{"Foo":"O'Reilly"}"""u8.ToArray();
+                context.AddRange(
+                    new ByteArrayJsonEntity { Id = 1, Json = json, Jsonb = json },
+                    new ByteArrayJsonEntity
+                    {
+                        Id = 2,
+                        Json = """{"Foo":"Bar"}"""u8.ToArray(),
+                        Jsonb = """{"Foo":"Bar"}"""u8.ToArray()
+                    });
+                await context.SaveChangesAsync();
+
+                var parameterQuery = context.Set<ByteArrayJsonEntity>().Where(e => e.Jsonb == json);
+                var parameterResult = async ? await parameterQuery.SingleAsync() : parameterQuery.Single();
+                Assert.Equal(1, parameterResult.Id);
+
+                Fixture.TestSqlLoggerFactory.Clear();
+
+                var constantQuery = context.Set<ByteArrayJsonEntity>().Where(e => e.Jsonb == EF.Constant(json));
+                var constantResult = async ? await constantQuery.SingleAsync() : constantQuery.Single();
+                Assert.Equal(1, constantResult.Id);
+
+                AssertSql(
+                    """
+SELECT b."Id", b."Json", b."Jsonb"
+FROM "ByteArrayJsonEntities" AS b
+WHERE b."Jsonb" = '{"Foo":"O''Reilly"}'
+LIMIT 2
+""");
+            });
+    }
+
+    public class ByteArrayJsonEntity
+    {
+        public int Id { get; set; }
+        public byte[] Json { get; set; } = null!;
+        public byte[] Jsonb { get; set; } = null!;
+    }
+
     public class JsonTranslationsQueryNpgsqlFixture : JsonTranslationsQueryFixtureBase, ITestSqlLoggerFactory
     {
         protected override ITestStoreFactory TestStoreFactory
@@ -58,6 +154,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext con
             base.OnModelCreating(modelBuilder, context);
 
             modelBuilder.Entity<JsonTranslationsEntity>().Property(e => e.JsonString).HasColumnType("jsonb");
+
+            modelBuilder.Entity<ByteArrayJsonEntity>(b =>
+            {
+                b.ToTable("ByteArrayJsonEntities");
+                b.Property(e => e.Id).ValueGeneratedNever();
+                b.Property(e => e.Json).HasColumnType("json");
+                b.Property(e => e.Jsonb).HasColumnType("jsonb");
+            });
         }
 
         protected override string RemoveJsonProperty(string column, string property)

test/EFCore.PG.Tests/Storage/NpgsqlTypeMappingSourceTest.cs

@@ -313,6 +313,36 @@ public void Json_primitive_collection()
         Assert.Same(typeof(int), elementMapping.ClrType);
     }
 
+    [Theory]
+    [InlineData("jsonb")]
+    [InlineData("json")]
+    public void Json_byte_array(string storeType)
+    {
+        // byte[] mapped to json/jsonb is the raw UTF-8 JSON document, not a primitive collection of bytes
+        var mapping = CreateTypeMappingSource().FindMapping(typeof(byte[]), storeType);
+        Assert.IsType<NpgsqlJsonTypeMapping>(mapping);
+        Assert.Equal(storeType, mapping.StoreType);
+        Assert.Same(typeof(byte[]), mapping.ClrType);
+        Assert.Null(mapping.ElementTypeMapping);
+    }
+
+    [Fact]
+    public void Json_byte_array_configured_as_primitive_collection()
+    {
+        var modelBuilder = CreateModelBuilder();
+        modelBuilder.Entity<ByteArrayEntity>().PrimitiveCollection(e => e.Bytes).HasColumnType("jsonb");
+        var model = modelBuilder.FinalizeModel();
+
+        var mapping = model.FindEntityType(typeof(ByteArrayEntity))!.FindProperty(nameof(ByteArrayEntity.Bytes))!
+            .GetRelationalTypeMapping();
+        Assert.Equal("jsonb", mapping.StoreType);
+
+        var elementMapping = (RelationalTypeMapping)mapping.ElementTypeMapping;
+        Assert.NotNull(elementMapping);
+        Assert.Equal("smallint", elementMapping.StoreType);
+        Assert.Same(typeof(byte), elementMapping.ClrType);
+    }
+
     [Theory]
     [InlineData(typeof(JsonDocument[]))]
     [InlineData(typeof(List<JsonDocument>))]
@@ -464,6 +494,12 @@ private class DummyType;
 
     private class UnknownType;
 
+    private class ByteArrayEntity
+    {
+        public int Id { get; set; }
+        public byte[] Bytes { get; set; }
+    }
+
     protected IModel CreateEmptyModel()
         => CreateModelBuilder().Model.FinalizeModel();
 

test/EFCore.PG.Tests/Storage/NpgsqlTypeMappingTest.cs

@@ -3,6 +3,7 @@
 using System.Net;
 using System.Net.NetworkInformation;
 using System.Numerics;
+using System.Text;
 using System.Text.Json;
 using Microsoft.EntityFrameworkCore.Design.Internal;
 using Microsoft.EntityFrameworkCore.Storage.Json;
@@ -871,6 +872,21 @@ public void GenerateSqlLiteral_returns_jsonb_string_literal()
     public void GenerateSqlLiteral_returns_json_string_literal()
         => Assert.Equal("""'{"a":1}'""", GetMapping("json").GenerateSqlLiteral("""{"a":1}"""));
 
+    [Theory]
+    [InlineData("json")]
+    [InlineData("jsonb")]
+    public void GenerateSqlLiteral_returns_json_byte_array_literal(string storeType)
+    {
+        var mapping = GetMapping(typeof(byte[]), storeType);
+
+        Assert.Equal("""'{"a":1}'""", mapping.GenerateSqlLiteral("""{"a":1}"""u8.ToArray()));
+        Assert.Equal(
+            "'{\"Name\":\"O''Reilly \u00e9\"}'",
+            mapping.GenerateSqlLiteral(Encoding.UTF8.GetBytes("{\"Name\":\"O'Reilly \u00e9\"}")));
+        Assert.Equal("'[1,2]'", mapping.GenerateSqlLiteral("[1,2]"u8.ToArray()));
+        Assert.Equal("'null'", mapping.GenerateSqlLiteral("null"u8.ToArray()));
+    }
+
     [Fact]
     public void GenerateSqlLiteral_returns_jsonb_object_literal()
     {