JSON and Array types for sqlmodel about sqlmodel

[Chunkford]

See if this works

from typing import List
from sqlmodel import Field, ARRAY, SQLModel, create_engine, Column, Float


class ArrayData(SQLModel, table=True):
    uuid: str = Field(primary_key=True)
    array: List[float] = Field(sa_column=Column(ARRAY(Float)))

[mv-yurchenko]

And I think examples with JSON and Array types will be very useful

[mv-yurchenko]

@raphapassini Absolutely need it

[cworkschris]

And I think examples with JSON and Array types will be very useful

Exactly - I'm confused how the Array solution of specific typing translates to JSON (the more experimentation I do the more confused I get 🙃️ pydantic newb here)

[raphapassini]

Hey guys, I guess this is a good topic for the documentation, @tiangolo what do you think? I can send a PR with some docs if you want to :)

[mbanerjeepalmer]

Since this is one of relatively few search results for how to use JSON, I should mention that got it to work by importing JSON from sqlmodel and passing that in: event: str = Field(sa_type=JSON).

Not sure if this is correct, though. (Note that my JSON is plain text, not jsonb.)

[yngtdd]

How should the ARRAY type be handled?

from sqlmodel import (
    Field, ARRAY, SQLModel, create_engine
)


class ArrayData(SQLModel, table=True):
    """Array ORM model"""
    
    uuid: str = Field(primary_key=True)
    array: ARRAY = [1.0]
        
    class Config:
        arbitrary_types_allowed = True

This seems to set the table alright:

ArrayData.__table__

>>> Table('arraydata', MetaData(), Column('uuid', AutoString(), table=<arraydata>, primary_key=True, nullable=False), Column('array', NullType(), table=<arraydata>, default=ColumnDefault([1.0])), schema=None)

but when I go to create the engine:

engine = create_engine("postgresql://toddyoung@localhost/postgres")

SQLModel.metadata.create_all(engine)

I get

---------------------------------------------------------------------------
CompileError                              Traceback (most recent call last)
~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/compiler.py in visit_create_table(self, create, **kw)
   4158             try:
-> 4159                 processed = self.process(
   4160                     create_column, first_pk=column.primary_key and not first_pk

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/compiler.py in process(self, obj, **kwargs)
    488     def process(self, obj, **kwargs):
--> 489         return obj._compiler_dispatch(self, **kwargs)
    490 

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py in _compiler_dispatch(self, visitor, **kw)
     81         else:
---> 82             return meth(self, **kw)
     83 

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/compiler.py in visit_create_column(self, create, first_pk, **kw)
   4192 
-> 4193         text = self.get_column_specification(column, first_pk=first_pk)
   4194         const = " ".join(

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/dialects/postgresql/base.py in get_column_specification(self, column, **kwargs)
   2508         else:
-> 2509             colspec += " " + self.dialect.type_compiler.process(
   2510                 column.type,

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/compiler.py in process(self, type_, **kw)
    520     def process(self, type_, **kw):
--> 521         return type_._compiler_dispatch(self, **kw)
    522 

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py in _compiler_dispatch(self, visitor, **kw)
     81         else:
---> 82             return meth(self, **kw)
     83 

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/compiler.py in visit_null(self, type_, **kw)
   4739     def visit_null(self, type_, **kw):
-> 4740         raise exc.CompileError(
   4741             "Can't generate DDL for %r; "

CompileError: Can't generate DDL for NullType(); did you forget to specify a type on this Column?

The above exception was the direct cause of the following exception:

CompileError                              Traceback (most recent call last)
<ipython-input-7-8e8474ad75f1> in <module>
      1 engine = create_engine("postgresql://toddyoung@localhost/postgres")
      2 
----> 3 SQLModel.metadata.create_all(engine)

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/schema.py in create_all(self, bind, tables, checkfirst)
   4738         if bind is None:
   4739             bind = _bind_or_error(self)
-> 4740         bind._run_ddl_visitor(
   4741             ddl.SchemaGenerator, self, checkfirst=checkfirst, tables=tables
   4742         )

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/future/engine.py in _run_ddl_visitor(self, visitorcallable, element, **kwargs)
    340         # single transaction rather than COMMIT for each statement.
    341         with self.begin() as conn:
--> 342             conn._run_ddl_visitor(visitorcallable, element, **kwargs)
    343 
    344     @classmethod

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/engine/base.py in _run_ddl_visitor(self, visitorcallable, element, **kwargs)
   2080 
   2081         """
-> 2082         visitorcallable(self.dialect, self, **kwargs).traverse_single(element)
   2083 
   2084     @util.deprecated(

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py in traverse_single(self, obj, **kw)
    522             meth = getattr(v, "visit_%s" % obj.__visit_name__, None)
    523             if meth:
--> 524                 return meth(obj, **kw)
    525 
    526     def iterate(self, obj):

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/ddl.py in visit_metadata(self, metadata)
    844         for table, fkcs in collection:
    845             if table is not None:
--> 846                 self.traverse_single(
    847                     table,
    848                     create_ok=True,

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py in traverse_single(self, obj, **kw)
    522             meth = getattr(v, "visit_%s" % obj.__visit_name__, None)
    523             if meth:
--> 524                 return meth(obj, **kw)
    525 
    526     def iterate(self, obj):

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/ddl.py in visit_table(self, table, create_ok, include_foreign_key_constraints, _is_metadata_operation)
    888             include_foreign_key_constraints = None
    889 
--> 890         self.connection.execute(
    891             # fmt: off
    892             CreateTable(

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/future/engine.py in execute(self, statement, parameters, execution_options)
    278 
    279         """
--> 280         return self._execute_20(
    281             statement, parameters, execution_options or NO_OPTIONS
    282         )

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/engine/base.py in _execute_20(self, statement, parameters, execution_options)
   1581             )
   1582         else:
-> 1583             return meth(self, args_10style, kwargs_10style, execution_options)
   1584 
   1585     def exec_driver_sql(

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/ddl.py in _execute_on_connection(self, connection, multiparams, params, execution_options)
     75         self, connection, multiparams, params, execution_options
     76     ):
---> 77         return connection._execute_ddl(
     78             self, multiparams, params, execution_options
     79         )

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/engine/base.py in _execute_ddl(self, ddl, multiparams, params, execution_options)
   1348         dialect = self.dialect
   1349 
-> 1350         compiled = ddl.compile(
   1351             dialect=dialect, schema_translate_map=schema_translate_map
   1352         )

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/elements.py in compile(self, bind, dialect, **kw)
    487                     ).get_dialect()()
    488 
--> 489         return self._compiler(dialect, **kw)
    490 
    491     def _compile_w_cache(

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/ddl.py in _compiler(self, dialect, **kw)
     27         Dialect."""
     28 
---> 29         return dialect.ddl_compiler(dialect, self, **kw)
     30 
     31     def _compile_w_cache(self, *arg, **kw):

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/compiler.py in __init__(self, dialect, statement, schema_translate_map, render_schema_translate, compile_kwargs)
    452             if self.can_execute:
    453                 self.execution_options = statement._execution_options
--> 454             self.string = self.process(self.statement, **compile_kwargs)
    455 
    456             if render_schema_translate:

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/compiler.py in process(self, obj, **kwargs)
    487 
    488     def process(self, obj, **kwargs):
--> 489         return obj._compiler_dispatch(self, **kwargs)
    490 
    491     def __str__(self):

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/visitors.py in _compiler_dispatch(self, visitor, **kw)
     80 
     81         else:
---> 82             return meth(self, **kw)
     83 
     84     cls._compiler_dispatch = (

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/sql/compiler.py in visit_create_table(self, create, **kw)
   4167                     first_pk = True
   4168             except exc.CompileError as ce:
-> 4169                 util.raise_(
   4170                     exc.CompileError(
   4171                         util.u("(in table '%s', column '%s'): %s")

~/opt/anaconda3/lib/python3.8/site-packages/sqlalchemy/util/compat.py in raise_(***failed resolving arguments***)
    205 
    206         try:
--> 207             raise exception
    208         finally:
    209             # credit to

CompileError: (in table 'arraydata', column 'array'): Can't generate DDL for NullType(); did you forget to specify a type on this Column?

[aedify-swi]

Have you tried setting a model config?
Link to pydantic docs: https://pydantic-docs.helpmanual.io/usage/types/#arbitrary-types-allowed

from typing import Optional

from sqlmodel import ARRAY, JSON, Field, SQLModel


class Hero(SQLModel, table=True):
    id: Optional[int] = Field(default=None, primary_key=True)
    name: str
    secret_name: JSON
    age: Optional[int] = None

    class Config:
        arbitrary_types_allowed = True


Hero(name="Max", secret_name={"name": "bob"}, age=12)

Edit: Never mind, just tried it and it gets rid of the error but doesn't actually set the field (it's None).

[FalseDev]

ARRAY might need a generic type argument like so

array: ARRAY[float]

Try updating your code to:

from sqlmodel import (
    Field, ARRAY, SQLModel, create_engine
)


class ArrayData(SQLModel, table=True):
    """Array ORM model"""
    
    uuid: str = Field(primary_key=True)
    array: ARRAY[float] = [1.0]
        
    class Config:
        arbitrary_types_allowed = True

[mv-yurchenko]

@FalseDev
Tried it. Got following error:
sqlalchemy.exc.CompileError: (in table 'arraydata', column 'array'): Can't generate DDL for NullType(); did you forget to specify a type on this Column?

[yngtdd]

Thanks @Chunkford, that works!

[mv-yurchenko]

Thanks a lot for all!!!

[Spenhouet]

= Field(sa_column=Column(ARRAY(Float)))

I tried this but I can not get it to work:

CompileError: (in table 'block', column 'transactions'): Compiler <sqlalchemy.dialects.sqlite.base.SQLiteTypeCompiler object at 0x7f09aae2f690> can't render element of type ARRAY

I have the following field:

transactions: List[HexStr] = Field(sa_column=Column(ARRAY(String)))

Where HexStr is just a custom pydantic type:

class HexStr(str):

    @classmethod
    def __get_validators__(cls):
        yield cls.validate

    @classmethod
    def validate(cls, v: Union[str, HexBytes]):
        if isinstance(v, HexBytes):
            return Web3.toHex(v)
        elif isinstance(v, str):
            return v
        else:
            raise NotImplementedError(f"Type {type(v)} unknown.")

I want a simple list of strings in the database cell. I can not find anything on that in the SQLModel documentation.

If I just do that:

transactions: List[HexStr]

Then I'm getting this exception:

InterfaceError: (sqlite3.InterfaceError) Error binding parameter 17 - probably unsupported type.
[SQL: INSERT INTO block ...

@Chunkford, @tiangolo any idea?

[Chunkford]

@Spenhouet I think you might find it's because SQLite doesn't support the ARRAY data type.