treeform / jsony Goto Github PK

There could be a parameter that accepts a format string. If this parameter isn't set and a DateTime field is encountered then raise an exception.

This would help to get rid of a lot of data structure wrangling in code that uses this library and uses DateTime fields.

Is there a way of handling a parameter that may be null?
To support a JSON that may come as:

{ "key": 2 }

or as

{ "key": null }

I already tried using Option this way:

type 
  Test = object
    key: Option[int]

This works for the first JSON but the second one gives the following error

Error: unhandled exception: Number expected. At offset: 6 [JsonError]

Hey
i wonder if there's way to just throw better exception , with field/type name

example:

type Entry1 = object
  username: string
  password: string

serializing this: "username":true it just tells me : e: Expected " but got t instead. At offset: 17
expected result: Wrong type: username should be string
obviously I don't want to check everything manually..

I'm obviously doing something wrong, preparing to facepalm already, but a very simple test fails for me:

Nim 1.6.16, jsony 1.1.5, Mac OS 14.1 M1

import playdate/api
import jsony
import utils
import options

let kFileReadAny*: FileOptions = cast[FileOptions]({kFileRead, kFileReadData})

type TestConfig* = ref object of RootObj
  lastOpenedLevel*: Option[string]
  tiltAttitudeAdjustEnabled*: Option[bool]
  dPadInputMultiplier*: Option[float]

type TestObj = object
  a: int
  b: string
  c: bool
  d: float

proc saveJson*[T](value: T, path: string) {.raises:[].} =
  let someConfig = TestConfig(lastOpenedLevel: some("level1"), tiltAttitudeAdjustEnabled: some(true), dPadInputMultiplier: some(1.5))
  let someTestObj = TestObj(a: 1, b: "hello", c: true, d: 1.5)
  # next line fails with someConfig, someTestObj, and the input param I'm actually tryingto serialize, which is value
  let jsonString = someTestObj.toJson()
  # [..] more code

  Executing task simulator in /Users/ninovanhooff/PlaydateProjects/wheelsprung.worktrees/level-data/wheelsprung.nimble
  Verifying dependencies for [email protected]
    Reading official package list
   Checking for https://github.com/samdze/playdate-nim@#main
      Info: Dependency on https://github.com/samdze/playdate-nim@#main already satisfied
  Verifying dependencies for playdate@#main
    Reading official package list
   Checking for https://github.com/ninovanhooff/nim-chipmunk-playdate@any version
      Info: Dependency on https://github.com/ninovanhooff/nim-chipmunk-playdate@any version already satisfied
  Verifying dependencies for [email protected]
    Reading official package list
   Checking for jsony@any version
      Info: Dependency on jsony@any version already satisfied
  Verifying dependencies for [email protected]
   Building wheelsprung/wheelsprung using c backend
  Executing /Users/ninovanhooff/.nimble/bin/nim c --colors:on --noNimblePath -d:simulator -d:debug -d:NimblePkgVersion=0.2.0 --path:'/Users/ninovanhooff/.nimble/pkgs/playdate-#main' --path:/Users/ninovanhooff/.nimble/pkgs/chipmunk7-7.0.3 --path:/Users/ninovanhooff/.nimble/pkgs/jsony-1.1.5 -o:/Users/ninovanhooff/PlaydateProjects/wheelsprung.worktrees/level-data/wheelsprung /Users/ninovanhooff/PlaydateProjects/wheelsprung.worktrees/level-data/src/wheelsprung.nim
Hint: used config file '/Users/ninovanhooff/.choosenim/toolchains/nim-1.6.16/config/nim.cfg' [Conf]
Hint: used config file '/Users/ninovanhooff/.choosenim/toolchains/nim-1.6.16/config/config.nims' [Conf]
Hint: used config file '/Users/ninovanhooff/PlaydateProjects/wheelsprung.worktrees/level-data/config.nims' [Conf]
............................................................................................................................
/Users/ninovanhooff/PlaydateProjects/wheelsprung.worktrees/level-data/src/data_store/configuration.nim(14, 11) template/generic instantiation of `saveJson` from here
/Users/ninovanhooff/PlaydateProjects/wheelsprung.worktrees/level-data/src/common/json_utils.nim(22, 31) template/generic instantiation of `toJson` from here
/Users/ninovanhooff/.nimble/pkgs/jsony-1.1.5/jsony.nim(881, 11) template/generic instantiation of `dumpHook` from here
/Users/ninovanhooff/.nimble/pkgs/jsony-1.1.5/jsony.nim(801, 8) template/generic instantiation of `dumpKey` from here
/Users/ninovanhooff/.nimble/pkgs/jsony-1.1.5/jsony.nim(739, 15) Error: attempting to call undeclared routine: 'toJson'
     Error: Build failed for package: wheelsprung
        ... Execution failed with exit code 256
        ... Command: /Users/ninovanhooff/.nimble/bin/nim c --colors:on --noNimblePath -d:simulator -d:debug -d:NimblePkgVersion=0.2.0 --path:'/Users/ninovanhooff/.nimble/pkgs/playdate-#main' --path:/Users/ninovanhooff/.nimble/pkgs/chipmunk7-7.0.3 --path:/Users/ninovanhooff/.nimble/pkgs/jsony-1.1.5 -o:/Users/ninovanhooff/PlaydateProjects/wheelsprung.worktrees/level-data/wheelsprung /Users/ninovanhooff/PlaydateProjects/wheelsprung.worktrees/level-data/src/wheelsprung.nim
stack trace: (most recent call last)
/private/var/folders/ww/sx_qg7z51jz6bgr15k4_5y8m0000gn/T/nimblecache-3905360933/nimscriptapi_3772293278.nim(187, 16)
/Users/ninovanhooff/PlaydateProjects/playdate-nim/src/playdate/build/nimble.nim(74, 16) simulatorTask
/Users/ninovanhooff/PlaydateProjects/playdate-nim/src/playdate/build/utils.nim(22, 10) nimble
/Users/ninovanhooff/.choosenim/toolchains/nim-1.6.16/lib/system/nimscript.nim(273, 7) exec
/Users/ninovanhooff/.choosenim/toolchains/nim-1.6.16/lib/system/nimscript.nim(273, 7) Error: unhandled exception: FAILED: nimble -d:simulator -d:debug build --verbose [OSError]
     Error: Exception raised during nimble script execution

Dear Sir,

I can confirm that your jsony module compiles and works with latest devel compiler, at least the most important procs toJson() and fromJson(). It was easy to replace std/json with that.

The fact that extra json fields are ignored and missing json fields keep their default values, is very important for real life development, and most other json libs ignore that. Actually I had stopped the development of my CAD/EDA tool for a long time, due to the fact that each tiny modification of data structures invalidates all stored test datasets. So after switching to your tool, I may get again some more motivation to do at least some work on that project again.

I also tried nim-jaml recently, but it has some issues: flyx/NimYAML#130
And json from status-im has some interesting issues as well, at least with latest devel compiler.

For your jsony, I would like to have a pretty option. Std/json has a pretty proc, and status-im provides a pretty flag. That is very useful, as in CAD area, users may like to edit data files manually with an text editor.

For default values, Nim 2.0 allows to specify default values for object fields. It would be nice if that default would be used as well. Your hooks work fine as well, but documentation is not fully clear about the fact that only binary zero default is used actually. From Readme: "... missing json fields keep their default values."

type
  Line = object
    x1, y1, x2: float
    pi: float = 3.14159


proc newHook(line: var Line) =
  # Populates the object before its fully deserialized.
  line.pi = 3.14159

I hope we will get support for "Pretty and "Easy way to skip fields of an object" #50 soon, would be really useful.

Best regards,

Dr. Stefan Salewski

Hey,

I just want to store enums as ints,

wrapper.nim:

import jsony
import inner

type
    Enum = enum
        e1
        e2

echo fromJson("0", Enum)

inner.nim

import std/[parseutils]

proc parseHook*[T: enum](s: string, i: var int, v: var T) =
  var temp: int
  inc i, parseInt(s, temp, i)
  v = T temp

proc dumpHook*(s: var string, v: enum) =
  s.add $v.int

Expected behaviour

compiles successfully and considers enums as ints

Current output

/wrapper.nim(10, 14) template/generic instantiation of `fromJson` from here
/.nimble/jsony/jsony.nim(590, 4) 

Error: ambiguous call; both 

jsony.parseHook(s: string, i: var int, v: var T: enum) [proc declared in /.nimble/jsony/jsony.nim(406, 6)] 
and 
inner.parseHook(s: string, i: var int, v: var T: enum) [proc declared in inner.nim(3, 6)] 

match for: (string, int, Enum)

It's probably better to have the API hide s and i in

This can be done with a lightweight parser object

type JsonParser = object
  view: openarray[char]
  pos: int

proc parseHook*[T](p: var JsonParser, v: var seq[T])
proc parseHook*[T: enum](p: var JsonParser, v: var T)
proc parseHook*[T: object|ref object](p: var JsonParser, v: var T)
proc parseHook*[T](p: var JsonParser, v: var SomeTable[string, T])
proc parseHook*[T](p: var JsonParser, v: var SomeSet[T])
proc parseHook*[T: tuple](p: var JsonParser, v: var T)
proc parseHook*[T: array](p: var JsonParser, v: var T)
proc parseHook*[T: ref array](p: var JsonParser, v: var T)
proc parseHook*(p: var JsonParser, v: var JsonNode)
proc parseHook*(p: var JsonParser, v: var char)

The API would be clearer instead of the user asking themself what that i parameter does and whether it was important or not.
It also gives you the ability to evolve the internals to add new functionality like a File field #5 or mmap support for large json files.

beside newHook and renameHook, providing some pragmas like dontSerialize, defaultDeserialize, serializationKey for objects fileds is very useful and make better serialization/deserialization control, like:

type MyObj = object
  a {.dontSerialize.}: int
  b {.defaultDeserialize: 5.}: int
  c {.serializationKey: "_c".}: string

any plan for this?
can I work on it?

Hello,

It seems jsony json parser has issue parsing some floats with E scientific notation.

Here's the minimal exemple code to reproduce the issue:

import jsony
import std/json

let s = r"""[9e-8]"""
#echo s
echo parseJson(s) # std json parse works
echo fromJson(s) # Invalid float. At offset: 5 [JsonError]

Some running context:

• Nim Compiler Version 1.6.0 [Windows: amd64]
• jsony installed with nimble install jsony today (2021-11-04)
• throw in release or debug mode, no other compiler flags used

The Issue

The payload [9e-8] is a valid json and std/json or python json parser handle it without raising errors.
Jsony should be able to parse it successfully.

Please note that, for now, I only encoutered this error with this specific value 9e-8.
Other values parsed with jsony (even those in E scientific notation) are successfully parsed.

Regards.

learnning jsony and found error of the following code:

proc parseHook*(s: string, i: var int, v: var DateTime) =
  var str: string
  parseHook(s, i, str)
  v = parse(str, "yyyy-MM-dd hh:mm:ss")

var dt = """ "2020-01-01 00:00:00" """.fromJson(DateTime)

and I couldn't find the parse proc in the api.

Test case:

import jsony, tables

type Answer {.pure.} = enum
    A, B, C

let a = {Answer.A: "aaaa", Answer.B: "bbb"}.toTable

echo a.toJson()
echo a.toJson().fromJson(Table[Answer, string])
echo a.toJson().fromJson(a.type)

Output:

{"A":"aaaa","B":"bbb"}
{:}
{:}

Should be:

{"A":"aaaa","B":"bbb"}
{A: "aaaa", B: "bbb"}
{A: "aaaa", B: "bbb"}

My current use-case for json is to consume auto-generated test vectors from libraries in other languages.
If I forget to consume a field, it's because I forget to update my consumer, if I try to read a field that doesn't exist, it's because I forgot to dump it in my producer.

It would be nice to have a strict mode to prevent those.

Note that if you get the check behind if parser.requiresStrict boolean field that is 100% predictable the performance should be the same. Some read on branch prediction I wrote today in a very sensitive context if you're interested (as in a branch misprediction would cost 10% of the total procedure): supranational/blst#10 (comment)

I'm trying to transform a json object that has leading underscores in some keys. Is there anyway to deserialize those into an object besides using a JsonNode?

{ "_id": "someId" }

I can override 5 other dumpHooks, but secifically this one gives me nim ambiguous call.
It might be a nim error and not the fault of this library, and then it might make sense to at least document this hook with a NOTE about it not being overridable.

proc dumpHook*[T](s: var string, v: seq[T])

Whenever I do fromJson to an object/ref object that contains a tuple, I always seem to get Exception message: Expected [ but got { instead. At offset: 70 Exception type: [JsonError].

However, I have noticed that on parseHook*[T: tuple] on jsony.nim the proc is slightly identical to parseHook*[T: array] proc as seen below the parseHook*[T: tuple] proc. This means that tuples are treated as arrays.

The parseHook*[T: tuple] proc should treat named tuples as objects (By checking if tuple is named) and treat unnamed tuples as arrays instead.

Here is an example to reproduce this error message
https://play.nim-lang.org/#ix=3JzX

Adding support for std/critbits is easy

To be changed to

type
  SomeTable*[K, V] = Table[K, V] | OrderedTable[K, V] |
    TableRef[K, V] | OrderedTableRef[K, V] | CritBitTree[V]

Somewhat related to this issue, this would allow the ability to change the output of fromJson back to valid JSON.

Version: Jsony 1.1.3 + json_serialization 0.1.0 (as part of chronicles 0.10.2)

The core issue is, that chronicles uses a package json_serialization and also exports it. Unfortunately, both jsony and json_serialization define toJson(string) procs. That isn't further tragic, when I can just specify jsony.toJson for my own code. What follows though is that I still get the "ambiguous call" compiler issue, but now from within jsony. dom96 assumes that this issue might be related to the timing when nim's generics are being created. I myself have no idea and am completely out of my depth on this one.

These issues happen specifically when using "jsony.toJson()" while both jsony and chronicles are imported in a package. Here a minimal example:

import chronicles
import jsony

type A = object
  name: string

let a = A(name: "walumba")
echo jsony.toJson(a)

This will cause this error during compilation:

/home/isofruit/dev/testingrounds/src/testingrounds.nim(11, 18) template/generic instantiation of toJson from here
/home/isofruit/.nimble/pkgs/jsony-1.1.3/jsony.nim(830, 11) template/generic instantiation of dumpHook from here
/home/isofruit/.nimble/pkgs/jsony-1.1.3/jsony.nim(758, 8) template/generic instantiation of dumpKey from here
/home/isofruit/.nimble/pkgs/jsony-1.1.3/jsony.nim(696, 20) Error: ambiguous call; both writer.toJson(v: GenericParam, pretty: bool, typeAnnotations: bool) [proc declared in /home/isofruit/.nimble/pkgs/json_serialization-0.1.0/json_serialization/writer.nim(235, 6)] and jsony.toJson(v: T) [proc declared in /home/isofruit/.nimble/pkgs/jsony-1.1.3/jsony.nim(829, 6)] match for: (string)
The terminal process "/usr/bin/bash '-c', 'nim c -r -d:normDebug --threads:on ~/dev/testingrounds/src/testingrounds.nim'" terminated with exit code: 1.

I managed to fix this manually by manipulating jsony.nim line 695-697 to explicitly use jsony.toJson:

template dumpKey(s: var string, v: string) =
  const v2 = jsony.toJson(v) & ":"
  s.add v2

However, naturally that isn't really long-term viable for me as this change will be gone with the next jsony version. I'm also not sure though whether this actually should be a PR to change the jsony source code or not.

I'm semi-certain it's an indicator for an issue at large, though I don't understand really what it is, even though I managed to find a fix. Thus I'm not sure whom exactly I should inform about the matter. I settled on opening an issue here because I managed to solve it by manipulating jsony's code. I'd ask for your advice on the matter whether this change to the 3 lines should be incorporated into jsony and whether to also inform the guys from status about the issue.

As of now, jsony doesn't recognize surrogate pairs and just keeps them as-is, resulting in invalid UTF-8 characters:

import std/json
import pkg/jsony

type
  TestObj = object
    content: string

let
  # A string with 🔒 emoji encoded both as normal UTF-8 and as a surrogate pair
  raw = """{"content":"\uD83D\uDD12🔒"}"""
  parsed = raw.fromJson(TestObj)
  parsedStd = parseJson(raw).to(TestObj)

echo "jsony - ", parsed.content
echo "std/json - ", parsedStd.content

Both std/json and nim-json-serialization handle surrogates, so I think jsony should also support them.

Found out about this bug from a discussion with @zedeus.

Hi
Pretty new to NIM so It's likely that I'm missing some information that I can't see in the doc but is there. Whit this premisis here my question:
I have a JSON message received and I need to convert it to an object between several ones, for example "MessageType1" and "MessageType2"; the information if the JSON message is of type1 or type2 is written inside it, in a specific string field.
How can I convert the JSON message to the right object with jsony in the fastest way?

Thanks in advance

😄 😄 😄

I was trying to parse super simple json with std/json. Was a totally a nightmare.

{ 
  "foo" : "bar",
  "baz": ["a", "b", "c"]
}

into:

type
  Foo = object
    foo: string
    baz: seq[string]

You'd think this would be THE EASIEST thing to do. But no. Weird error, need to make seq[string] a JsonObject, then cycle through, using a .toStr() on each item. Totally weird. Also, breaks when you send to a proc (I guess because it's a bizzaro macro).

Worked first time in jsony, as you logically expect it to. Deals with missing fields. Etc.

This should replace std/json

Thank you!!!

JSON crashes on this JSON and I'm not sure why.

/home/user/.nimble/pkgs/jsony-1.1.3/jsony.nim(547) fromJson
/home/user/.nimble/pkgs/jsony-1.1.3/jsony.nim(429) parseHook
/home/user/.nimble/pkgs/jsony-1.1.3/jsony.nim(347) parseObjectInner
/home/user/.nimble/pkgs/jsony-1.1.3/jsony.nim(47) parseHook
Error: unhandled exception: Expected " but got { instead. At offset: 1048 [JsonError]

I'm using Nim 1.6.10 and JSONy 1.1.3 and I'm loading the JSON from a file with readFile and just passing the variable to fromJson.
I only added two simple hooks.

proc parseHook*(s: string, i: var int, v: var DateTime) =
  var str: string
  parseHook(s, i, str)
  v = parse(str, "yyyy-MM-dd'T'hh:mm:ss'.'fffzzz", utc())

proc renameHook*(v: var PreviewCard, fieldName: var string) =
  if fieldName == "type":
    fieldName = "card_type"

It doesn't seem like jsony supports loading from a file, which a user needs space to hold 2x the json size in memory.

toJson proc accepts optional pretty bool parameter to pretty-print output with newlines and indents.

So, if I

1. define a case object
2. use the case object as key in a Table
3. serialize the table
4. deserialize the table

The code fails at step 4 because it represents the key as a json object.

In example below, the generated json is:
{{"kind":"TermS","term":42}:42}

Can user expect the default json ser/deser to cover such a use case? If not, what's the correct way to use the library so that roundtripping works?

import jsony, tables, hashes

type SymbolKind* {.pure.} = enum
  TermS

type Symbol* = object
  case kind*: SymbolKind
  of SymbolKind.TermS:
    term*: int

proc hash*(x: Symbol): Hash =
  var h: Hash = 0
  h = h !& hash(x.kind)
  case x.kind 
  of TermS:
    h = h !& hash(x.term)
  else:
    discard
  return !$h

proc `==`*(a,b: Symbol) : bool = 
  return a.kind == b.kind and a.term == b.term

var t : Table[Symbol, int]

t[Symbol(kind: TermS, term: 42)] = 42

let str = t.toJson()
echo str
discard str.fromJson(Table[Symbol, int])

It seems that the javascript and c++ test are just compiled and not ran in the github actions.

I'm trying to isolate a bug with distinct type and create a minimal repro but I ended up getting stuck in another issue:

import jsony

type
  SecretWord = distinct uint64

  CryptographicType = object
    data: array[4, SecretWord]

  TestVector[T] = object
    vectors: seq[T]

proc parseHook(src: string, pos: var int, value: var CryptographicType) =
  var str: string
  parseHook(src, pos, str)
  discard "value.fromHex(str)"

let s = """
{
	"vectors": [{
		"data": "0x2523648240000001ba344d80000000086121000000000013a700000000000013"
	}]
}
"""
let v = s.fromJson(TestVector[CryptographicType])

According to https://jsonlint.com/ this is valid JSON but somehow I get

/[...]/Programming/Nim/constantine/build/jsony_distinct.nim(24) jsony_distinct
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(434) fromJson
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(318) parseHook
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(170) parseHook
/[...]/Programming/Nim/constantine/build/jsony_distinct.nim(14) parseHook
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(44) parseHook
Error: unhandled exception: Expected ". At offset: 15 [JsonError]

End goal

I'll open an issue if this ends up being an issue in jsony

I am trying to replace nim-json-serialization for my test vectors, it brings many dependencies and one is crashing my CI at the moment for a git clone issue: https://github.com/mratsim/constantine/pull/155/checks?check_run_id=1865342674#step:15:60 and unfortunately there is no easy way out of it until Nimble supports task level dependencies (it's a library tested with nim-json-serialization that is brought) (nim-lang/nimble#482).

I use distinct types in there but they are all caught by the 2 readValue in nim-json-serialization (https://github.com/mratsim/constantine/blob/c4a2dee/tests/t_ec_sage_template.nim#L95-L178) and example file (https://github.com/mratsim/constantine/blob/c4a2dee/tests/vectors/tv_BN254_Nogami_scalar_mul_G1.json). Somehow at the moment jsony is trying to parseHook those.

/[...]/Programming/Nim/constantine/tests/t_ec_sage_bls12_381.nim(18, 28) template/generic instantiation of `run_scalar_mul_test_vs_sage` from here
/[...]/Programming/Nim/constantine/tests/t_ec_sage_template.nim(186, 26) template/generic instantiation of `loadVectors` from here
/[...]/Programming/Nim/constantine/tests/t_ec_sage_template.nim(172, 19) template/generic instantiation of `fromJson` from here
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(434, 4) template/generic instantiation of `parseHook` from here
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(318, 20) template/generic instantiation of `parseHook` from here
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(170, 14) template/generic instantiation of `parseHook` from here
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(318, 20) template/generic instantiation of `parseHook` from here
/[...]/.nimble/pkgs/jsony-0.0.5/jsony.nim(197, 14) Error: type mismatch: got <string, int, SecretWord>
but expected one of: 
proc parseHook(s: string; i: var int; v: var JsonNode)
  first type mismatch at position: 3
  required type for v: var JsonNode
  but expression 'value' is of type: SecretWord
proc parseHook(s: string; i: var int; v: var SomeFloat)
  first type mismatch at position: 3
  required type for v: var SomeFloat
  but expression 'value' is of type: SecretWord

How does it compare to Araq/packedjson ?

Sometimes it would be useful to parse 0 1 int as bool.

Say I have an object like this:

  Player = ref object
    username: string
    ready: bool
    conn: WebSocket
    curLevel: JsonNode
    invadingUsername: string
    heistSuccess: bool
    id: uint64

that I want to dump to json, but don't want the conn to be in the dumped json. What if jsony had some kind of skipHook you could define to describe what fields to skip serializing?

This is valid json:

[
	["a", "b", "c"]
]

however, jsony cant parse it.

import jsony
type abc = object
    a : string
    b : string
    c : string
let s = "[["a", "b", "c"]]"
let v = s.fromJson(abc)
echo v

however, this library wasn't made with the json spec in mind.

This should return an array of abc.

Code:

import jsony, tables
let t = {100: "aaa"}.toTable
echo t.toJson

Actual:

{100:"aaa"}

^^^ json key uses quotes

Expected:

{"100":"aaa"}

Heya, I'm currently working on a project and decided to use jsony, but it seems that there's an issue with toJson as shown with this error:
/home/runner/.nimble/pkgs2/jsony-1.1.5-6aeb83e7481ca8686396a568096054bc668294df/jsony.nim(739, 15) Error: attempting to call undeclared routine: 'toJson'

I'm unable to make a minimal reproducible example despite my efforts, so I simply added a step to my workflow to demonstrate the issue: https://github.com/Luyten-Orion/Supernovae/actions/runs/9423485799/job/25961966047#step:4:82

The code that causes the issue seems to be here, but I'll also post the snippet for convenience: https://github.com/Luyten-Orion/Supernovae/blob/main/src/supernovae/api.nim#L11-L12

template respond[T](request: Request, code: int, headers: HttpHeaders, body: T) =
  request.respond(code, headers, toJson[T](body))

In a minute I'll create a PR to add the necessary forward declare to alleviate the issue.

The readme has many reference to the following API

type Entry1 = object
  color: string
var s = """{"extra":"foo"}"""
var v = fromJson[Entry1](s)
doAssert v.color == ""

But the library is expecting var v = s.fromJson(Entry1)

I have a situation where JSON strings omit default values. For instance in the code below, if interval is omitted or less than 0, it should be set to 100. What can I do with dumpHook/skipHook to omit default values when I serialize my objects?

import jsony, std/json, std/unittest

type
  Thing = object
    name*: string
    interval*: int

proc newHook*(x: var Thing) =
  if x.interval == 0:
    x.interval = 100

test "defaultvalue":
  let original = """{"name":"bob"}"""
  let thing = original.fromJson(Thing)
  check thing.interval == 100
  let serialized = thing.toJson()
  check parseJson(serialized) == parseJson(original)

test "specificvalue":
  let original = """{"name":"bob", "interval": 200}"""
  let thing = original.fromJson(Thing)
  check thing.interval == 200
  let serialized = thing.toJson()
  check parseJson(serialized) == parseJson(original)

The output of the above is:

parseJson(serialized) was {"name":"bob","interval":100}
parseJson(original) was {"name":"bob"}
[FAILED] defaultvalue
[OK] specificvalue

Looking for a faster approach to walking nested JSON data with an arbitrary structure. The stdlib json library is a bit slow. Can jsony be used to walk nested JSON data without specifying a type?

This won't work without also importing json, and I'm not sure if that is the correct approach in Nim:

import jsony
let s = readFile("test.json")
let js = s.fromJson()

for j in js:
  echo j

When converting from an object to JSON, if a field is an Option type, then I want to leave that field out of the JSON. The reason is to save space and thereby reduce IO because I'm saving JSON to the DB.

I can understand that you typically want to include the field with a null value, so a setting to optionally leave out null fields would be great.

import jsony
echo {"aaa": 100}.toJson()
# {"aaa":100}
echo {100: "aaa"}.toJson()
# [[100,"aaa"]]

Probably it is ok, but a bit inconsistent

Hey @treeform. Thanks for your great packages.

Example

Consider a typical Nim user who reads:

proc parseHook*[T](s: string, i: var int, v: var SomeSet[T]) =

proc dumpHook*[T](s: var string, v: SomeSet[T]) =

Expectation

They think that jsony does not support the built-in set type, because they're familiar with the definition in std/sets of just:

  SomeSet*[A] = HashSet[A] | OrderedSet[A]

Reality

jsony does support the set type, it's just that since commit ebee42e, jsony exports its own SomeSet type that includes set:

Discussion

Would it be better to go back to this instead?

proc parseHook*[T](s: string, i: var int, v: var (SomeSet[T]|set[T]))

This would also allow jsony to be used again with code that uses SomeSet. For example, this:

import std/sets
import pkg/jsony

proc foo[A](s: SomeSet[A]) =
  discard

compiled with jsony 1.0.5 and earlier, but doesn't compile with jsony 1.1.0:

/tmp/foo.nim(4, 16) Error: ambiguous identifier: 'SomeSet' -- use one of the following:
  sets.SomeSet: SomeSet
  jsony.SomeSet: SomeSet

This is why I started creating this issue - I saw exactly this breakage when upgrading jsony.

Sure, it's easy for that user to replace SomeSet with sets.SomeSet everywhere in their existing code. But that doesn't seem like a great fix.

One alternative is to rename the new exported type, but that's probably less obvious, and it might be hard to find a good name.

It's also not immediately obvious to every Nim user whether something like jsony.SomeSet[string] even compiles, because set[string] is not a valid type.

Imagine i want to code a json minifier using jsony, here is a minimal example:

import jsony
import os

when isMainModule:
    let params = commandLineParams()
    if len(params) != 2:
        echo "Missing parameters <input> <output>"
        quit QuitFailure

    let input = params[0]
    let output = params[1]
    let content = readFile(input)

    try:
        output.writeFile toJson(content.fromJson())
    except [IOError, JsonError]:
        echo "Failed minification !"
        output.writeFile content

If you feed this simple program with this json:

[
    {
        "0": true,
        "1": true,
        "2": "MIUWITA(.3#Cu@50Cp>gbyr$",
        "4": -255861533,
        "5": "tf BIf_&\\291dDAaSd>(#T+'s_)3 HwZ$_I>sj",
        "6": false,
        "7": 687550500.0598536,
        "9": 1770799904,
        "17": "yl/C_oV*[l3O",
        "VRBkw": 1219049398.2606015,
        "8uY": {
            "3": true,
            "5": "snm-A<vFU~+73@W\\r|\"pnJkR(&*DQ|f\"$KxN["
        }
    }
]

You get back an invalid json:

[{"0":true,"1":true,"2":"MIUWITA(.3#Cu@50Cp>gbyr$","4":-255861533,"5":"tf BIf_&\\291dDAaSd>(#T+'s_)3 HwZ$_I>sj","6":false,"7":687550500.0598536,"9":1770799904,"17":"yl/C_oV*[l3O","VRBkw":1219049398.2606015,"8uY":{"3":true,"5":"snm-A<vFU~+73@W\\r|"pnJkR(&*DQ|f"$KxN["}}]

Notice the double quotes in this string that are not escaped correctly "snm-A<vFU~+73@W\\r|"pnJkR(&*DQ|f"$KxN[".

An equivalent python script works just fine:

import json, argparse, codecs, shutil
from collections import OrderedDict

def run(args):
    try:
        with codecs.open(args['in'], "r", encoding="UTF-8") as data_file:
            data = json.load(data_file, object_pairs_hook=OrderedDict)
            with open(args['out'], 'w') as file:
                x = json.dumps(data, separators=(',', ':'))
                file.write(x)
    except (TypeError, ValueError):
        shutil.copyfile(args['in'], args['out'])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description = '')
    parser.add_argument('in', help='input file')
    parser.add_argument('out', help='name of output file')
    args = vars(parser.parse_args())

    run(args)

Hi

What's the best way to omit serializing keys with null values?
Should I write a dumpHook or is there a better way?

Example:

type
    Foo = ref object
        bar: string
        baz: int

let a = Foo(
    bar: "bar"
)

# current
echo a.toJson()
"{"bar":"bar", "baz": null}"

# desired
echo a.toJson()
"{"bar":"bar"}"

Heyho!
I'm using jsony in a web-application. There I want to use it to serialize an de-serialize JSON strings into objects, specifically of norm-objects (an ORM) which is filled with data coming from a database.
Due to the way norm handles Datetimes, I had to implement a custom DateTime type that borrows/reimplements DateTime's functionality, that appears to cause some problems.

What I found is that, when doing so, jsony drops part of the serialization string silently, without an error message.
Here is a minimum example that demonstrates the issue when I run it on "https://play.nim-lang.org/".

import jsony
import options
import times

type DjangoDateTime* = distinct DateTime
proc format*(x: DjangoDateTime, f: string, loc: DateTimeLocale = DefaultLocale): string =
    let trueDt = x.DateTime 
    result = trueDt.format(f, loc)
    
proc dumpHook*(s: var string, value: DjangoDateTime) =
    s = value.format("yyyy-MM-dd HH:mm:ss'.'ffffff")

proc now*(): DjangoDateTime = DjangoDateTime(times.now())


type 
    A = object
        nameA: string
        datetimeA: DjangoDateTime


var a = A(nameA: "The name of A", datetimeA: now())
echo a.toJson()

This should print out {"nameA": "The name of A", "datetimeA": "<FORMATTED CURRENT DATETIME>"}.
Instead it prints out <FORMATTED CURRENT DATETIME>}.
It drops the parts of the string that come before DjangoDateTime is converted into a string.

I am not quite sure what is happening here, though I can only assume that it's DjangoDateTime causing this. How, I'm not sure.

Quote(") in string is supported in fromJson(), but not in toJson().

var s = """ "quote\"inside" """
var v = s.fromJson(string)
echo v
echo v.toJson.fromJson(string)

Output:

quote"inside
quote

Fix:

proc dumpHook*(s: var string, v: string) =
  when nimvm:
    s.add '"'
    for c in v:
      case c:
      of '\\': s.add r"\\"
      of '\b': s.add r"\b"
      of '\f': s.add r"\f"
      of '\n': s.add r"\n"
      of '\r': s.add r"\r"
      of '\t': s.add r"\t"
      of '"': s.add r"\"""
      else:
        s.add c
    s.add '"'
  else:
    # Its faster to grow the string only once.
    # Then fill the string with pointers.
    # Then cap it off to right length.
    var at = s.len
    s.setLen(s.len + v.len*2+2)

    var ss = cast[ptr UncheckedArray[char]](s[0].addr)
    template add(ss: ptr UncheckedArray[char], c: char) =
      ss[at] = c
      inc at
    template add(ss: ptr UncheckedArray[char], c1, c2: char) =
      ss[at] = c1
      inc at
      ss[at] = c2
      inc at

    ss.add '"'
    for c in v:
      case c:
      of '\\': ss.add '\\', '\\'
      of '\b': ss.add '\\', 'b'
      of '\f': ss.add '\\', 'f'
      of '\n': ss.add '\\', 'n'
      of '\r': ss.add '\\', 'r'
      of '\t': ss.add '\\', 't'
      of '"': ss.add '\\', '"'
      else:
        ss.add c
    ss.add '"'
    s.setLen(at)

Output:

quote"inside
quote"inside