nim-works / cps

Continuation-Passing Style for Nim 🔗

nim-works

cps's Issues

symbol signatures need a custom impl

Basically, we should trust no one and instead use something like line info to determine our own identity. I'm looking at hashes rewriting symbols and I don't trust what I see.

tuple unpacking does not work

that's worth an issue

unsupported `a, b, c: int = 5` syntax in proc params

fixing tock in a typed world

The title makes this sound interesting, sorry!

I'm finally looking at this stuff and here's what I found when trying to see
why tests/tock.nim doesn't work. It looks like there are two issues, one
of which is probably not too hard to solve; the other... I dunno.

Here's the test:

import cps
import cps/eventqueue

proc tock(name: string; ms: int) {.cps: Cont.} =
  var count = 10
  while 0 < count:
    dec count
    sleep ms
    echo name, " ", count

And here's how I rewrote the output to make it work:

import cps
import cps/eventqueue

type
  env_33554675 = ref object of Cont
    name_33554692: string
    ms_33554701: int
    count_33554982: int

proc loop_33554999(continuation: Cont): Cont
proc after_33555022(continuation: Cont): Cont
proc loop_33554999(continuation: Cont): Cont =
  if 0 < env_33554675(continuation).count_33554982:
    dec env_33554675(continuation).count_33554982, 1
    #[
    # this is the original form...
    return sleep Cont:
      continuation.fn = after_33555022
      continuation,
        env_33554675(continuation).ms_33554701
    # so we xfrm to asgn/call, which is harmless, but surprising:
    ]#
    continuation.fn = after_33555022
    return sleep(continuation, env_33554675(continuation).ms_33554701)

proc after_33555022(continuation: Cont): Cont =
  # i guess maybe there's a phase issue with echo's varargs converter?
  # (i stripped the brackets)
  echo env_33554675(continuation).name_33554692, " ",
        env_33554675(continuation).count_33554982
  # another xfrm to asgn/call
  continuation.fn = loop_33554999
  return continuation

proc tock(continuation: Cont): Cont {.cpsCall.} =
  env_33554675(continuation).count_33554982 = 10
  # another xfrm to asgn/call
  continuation.fn = loop_33554999
  return continuation

proc tock(name: string; ms: int): Cont =
  result = env_33554675(fn: tock, ms_33554701: ms, name_33554692: name)

spawn tock("tick", 300)
spawn tock("tock", 700)

run()

proc params apparently untyped

I need to do a better job of keeping track of bugs like this one:

The .cps. proc parameters like ms in the tick-tock example come across as ident nodes and not symbols, which means that despite moving them into the continuation, we cannot subsequently do a trivial swap of their symbol nodes in the proc body with the symbol of our gensym'd continuation field. This is a bummer. Having a proper symbol lets us unambiguously match symbols that may share the same name (such as ident nodes may) with (only) the symbol we're trying to replace.

Any ideas, @Clyybber?

boxing of vars unimplemented

break in while messes up flow

import cps           
                     
type C = ref object of RootObj
  fn*: proc(c: C): C {.nimcall.}
                  
proc prim(s: string): C {.cpsMagic.} =
  echo s            
  return c         
                      
proc f(): C {.cps.} =
  cps prim("one")    
  while true:        
    cps prim("two")   
    break
    cps prim("three")
  cps prim("four")   
                  
var c = f()     
                    
while c != nil and c.fn != nil:
  c = c.fn(c)

Will end up printing two four two four two four forever

unexpected expr(env typesym) yields ICE

Looks like it's another case of a symbol rewrite blowing the compiler's little mind.

# something like
echo name, " ", count
# turns into
echo [env_19230210(continuation).name_19230227, " ", env_19230210(continuation).count_19230429]

@Clyybber, should I be writing this differently? I doubt a template would help but we could try it...

Hey, at least it's a short trace.

/home/adavidoff/git/cps/cps/environment.nim(215, 16) Error: internal error: expr(skType); unknown symbol
Traceback (most recent call last)
/home/adavidoff/git/Nim/compiler/nim.nim(117) nim
/home/adavidoff/git/Nim/compiler/nim.nim(82) handleCmdLine
/home/adavidoff/git/Nim/compiler/cmdlinehelper.nim(84) loadConfigsAndRunMainCommand
/home/adavidoff/git/Nim/compiler/main.nim(259) mainCommand
/home/adavidoff/git/Nim/compiler/main.nim(224) compileToBackend
/home/adavidoff/git/Nim/compiler/main.nim(100) commandCompileToC
/home/adavidoff/git/Nim/compiler/modules.nim(160) compileProject
/home/adavidoff/git/Nim/compiler/modules.nim(96) compileModule
/home/adavidoff/git/Nim/compiler/passes.nim(208) processModule
/home/adavidoff/git/Nim/compiler/passes.nim(86) processTopLevelStmt
/home/adavidoff/git/Nim/compiler/cgen.nim(1926) myProcess
/home/adavidoff/git/Nim/compiler/cgen.nim(973) genProcBody
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2686) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2459) genStmtList
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2650) expr
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(750) genCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(747) genAsgnCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(352) genPrefixCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(333) genParams
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(257) genArg
/home/adavidoff/git/Nim/compiler/cgen.nim(613) initLocExprSingleUse
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2656) expr
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(750) genCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(747) genAsgnCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(346) genPrefixCall
/home/adavidoff/git/Nim/compiler/cgen.nim(600) initLocExpr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2580) expr
/home/adavidoff/git/Nim/compiler/cgen.nim(1223) genProc
/home/adavidoff/git/Nim/compiler/cgen.nim(1166) genProcNoForward
/home/adavidoff/git/Nim/compiler/cgen.nim(1032) genProcAux
/home/adavidoff/git/Nim/compiler/cgen.nim(973) genProcBody
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2717) expr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1568) genAsgn
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(139) loadInto
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2691) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2515) downConv
/home/adavidoff/git/Nim/compiler/cgen.nim(600) initLocExpr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2676) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(1419) genObjConstr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2580) expr
/home/adavidoff/git/Nim/compiler/cgen.nim(1223) genProc
/home/adavidoff/git/Nim/compiler/cgen.nim(1166) genProcNoForward
/home/adavidoff/git/Nim/compiler/cgen.nim(1032) genProcAux
/home/adavidoff/git/Nim/compiler/cgen.nim(973) genProcBody
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2686) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2470) genStmtList
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2686) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2459) genStmtList
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2712) expr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(461) genReturnStmt
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2717) expr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1568) genAsgn
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(139) loadInto
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2685) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2459) genStmtListExpr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2717) expr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1568) genAsgn
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(139) loadInto
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2580) expr
/home/adavidoff/git/Nim/compiler/cgen.nim(1223) genProc
/home/adavidoff/git/Nim/compiler/cgen.nim(1166) genProcNoForward
/home/adavidoff/git/Nim/compiler/cgen.nim(1032) genProcAux
/home/adavidoff/git/Nim/compiler/cgen.nim(973) genProcBody
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2687) expr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(445) genIf
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2686) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2459) genStmtList
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2712) expr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(461) genReturnStmt
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2717) expr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1568) genAsgn
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(129) loadInto
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(747) genAsgnCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(352) genPrefixCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(333) genParams
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(257) genArg
/home/adavidoff/git/Nim/compiler/cgen.nim(613) initLocExprSingleUse
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2685) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2459) genStmtListExpr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2717) expr
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1568) genAsgn
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(139) loadInto
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2580) expr
/home/adavidoff/git/Nim/compiler/cgen.nim(1223) genProc
/home/adavidoff/git/Nim/compiler/cgen.nim(1166) genProcNoForward
/home/adavidoff/git/Nim/compiler/cgen.nim(1032) genProcAux
/home/adavidoff/git/Nim/compiler/cgen.nim(973) genProcBody
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2686) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2459) genStmtList
/home/adavidoff/git/Nim/compiler/ccgstmts.nim(1575) genStmts
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2648) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2303) genMagicExpr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(1071) genEcho
/home/adavidoff/git/Nim/compiler/cgen.nim(600) initLocExpr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2668) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2435) genArrayConstr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2682) expr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(799) genRecordField
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(761) genRecordFieldAux
/home/adavidoff/git/Nim/compiler/cgen.nim(600) initLocExpr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2650) expr
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(750) genCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(747) genAsgnCall
/home/adavidoff/git/Nim/compiler/ccgcalls.nim(346) genPrefixCall
/home/adavidoff/git/Nim/compiler/cgen.nim(600) initLocExpr
/home/adavidoff/git/Nim/compiler/ccgexprs.nim(2631) expr
/home/adavidoff/git/Nim/compiler/msgs.nim(585) internalErrorImpl
/home/adavidoff/git/Nim/compiler/msgs.nim(541) liMessage
/home/adavidoff/git/Nim/compiler/msgs.nim(408) handleError
/home/adavidoff/git/Nim/compiler/msgs.nim(397) quit

sugar for calling cps procs from cps

Now I need to do

proc foo(): Cont {.cps.} =
  echo "Hello"
  return bar()
  echo "There"

proc bar(): Cont {.cps.} =
  echo "I'm bar"

I'd like to drop the return from the line return bar, so it looks like a regular call.

After 'while' code flow wrong

import cps           
                     
type C = ref object of RootObj
  fn*: proc(c: C): C {.nimcall.}
                  
proc prim(s: string): C {.cpsMagic.} =
  echo s            
  return c          
                      
proc f(): C {.cps.} = 
  var a: int = 0
  while a < 3:       
    echo a
    cps prim("two")  
    inc a
  echo "bye"  
                     
var c = f()          
                     
while c != nil and c.fn != nil:
  c = c.fn(c)

Will print "bye" forever

_clyybber mode poised to spark a matrix of procedure signatures

We need a way to accept arguments to .cpsCall. procs but we currently have only the bootstrap and continuation versions; do we create more _clyybber procedures?

cps/eventqueue.nim(9, 8) Error: cannot open file: sorta

move cpsDebug from define

I just want to cpsDebug individual .cps. calls, obviously.

arc crashes compiler or results in move(result, result)

As per title, it doesn't work even if it makes it to runtime.

Block rewriting error when there are no explicit break

Found while messing with "shadow mission impossible"

import cps

type
  Cont* = ref object of RootObj
    fn*: proc(c: Cont): Cont {.nimcall.}

proc trampoline(c: Cont) =
  var c = c
  while c != nil and c.fn != nil:
    c = c.fn(c)

proc noop*(c: Cont): Cont {.cpsMagic.} = c

template doAssert(b: bool, msg: string) =
  if not b:
    raise newException(AssertionDefect, msg)

var r = 0
proc b() {.cps: Cont.} =
  inc r
  block:
    noop()
    echo "in block"
    inc r
  inc r
  echo "out block"

trampoline b()
doAssert r == 3, "expected 3, got: " & $r

Output:

=== .cps. on b(original)  === temp.nim(19, 0)
 19  proc b() =
 20    inc r, 1
 21    block:
 22      noop()
 23      echo ["in block"]
 24      inc r, 1
 25    inc r, 1
 26    echo ["out block"]
 27
storing type env_385876243
next type env_385876522
=== .cps. on b(transform) === temp.nim(19, 0)
 19
 20  type
 21    env_385876243 = ref object of Cont
 22
 23  proc blockBreak_385876482(continuation: Cont): Cont {.cpsLift.}
 24  proc afterCall_385876492(continuation: Cont): Cont {.cpsLift.}
 25  proc blockBreak_385876482(continuation: Cont): Cont {.cpsLift.} =
 26    ## saften at 21 of temp.nim
 27    inc r, 1
 28    echo "out block"
 29
 30  proc afterCall_385876492(continuation: Cont): Cont {.cpsLift.} =
 31    ## saften at 848 of cps.nim
 32    echo "in block"
 33    inc r, 1
 34
 35  proc b(continuation: Cont): Cont {.cpsCall.} =
 36    ## saften at 19 of temp.nim
 37    ## saften at 20 of temp.nim
 38    inc r, 1
 39    block:
 40      ## saften at 22 of temp.nim
 41      ## re-use the local continuation by setting the fn
 42      continuation.fn = afterCall_385876492
 43      return noop(continuation)
 44    ## add tail call for block-break proc
 45    ## verbatim tail call
 46    ## new tail call: blockBreak_385876482
 47    ## re-use the local continuation by setting the fn
 48    continuation.fn = blockBreak_385876482
 49    return continuation
 50    ## new tail call: blockBreak_385876482
 51    ## re-use the local continuation by setting the fn
 52    continuation.fn = blockBreak_385876482
 53    return continuation
 54
 55  proc b(): Cont =
 56    result = env_385876243(fn: b)
 57
Hint:  [Link]
Hint: 53164 lines; 1.007s; 76.262MiB peakmem; Debug build; proj: temp; out: temp [SuccessX]
Hint: temp  [Exec]
in block
temp.nim(16) temp
Error: unhandled exception: expected 3, got: 2 [AssertionDefect]

shim try expressions

More function pointers issue

import cps

type
  Cont* = ref object of RootObj
    fn*: proc(c: Cont): Cont {.nimcall.}

proc trampoline(c: Cont) =
  var c = c
  while c != nil and c.fn != nil:
    c = c.fn(c)

proc noop*(c: Cont): Cont {.cpsMagic.} = c

proc bar(i: int): proc(): int =
  result = proc(): int =
    i * 2

proc foo() {.cps: Cont.} =
  var i = 2
  noop()
  i = bar(i)()
  noop()
  doAssert i == 4

trampoline foo()

Compiler output:

=== .cps. on foo(original)  === stretch.nim(18, 0)
 18  proc foo() =
 19    var i = 2
 20    noop()
 21    i = bar(i)()
 22    noop()
 23    echo [i == 4]
 24
nnkVarSection	i: int = 2
stack trace: (most recent call last)
cps.nim(877, 21)         cps
cps.nim(863, 25)         cpsXfrm
cps.nim(799, 15)         cpsXfrmProc
cps.nim(583, 21)         saften
cps.nim(371, 26)         saften
cps.nim(339, 27)         splitAt
cps.nim(320, 15)         procScope
cps.nim(583, 21)         saften
cps.nim(368, 10)         saften
cps.nim(50, 21)          isCpsCall
cps.nim(43, 7)           getCallSym
stretch.nim(18, 14) template/generic instantiation of `cps` from here
cps.nim(43, 7) Error: unhandled exception: unknown node type: nnkCall [Defect]

Fixing this is easy, but I might have to rethink the basis of getCallSym, since calls in Nim don't just come from symbols, expressions can be called too.

refactor inheritance layer

Ideally, it's a runtime dialect.

need a shim for continuation results as rvalues

Compilation failure: interpretation requires too many iterations

The snippet below cause compilation to hang:

import cps           
                  
type C = ref object of RootObj
  fn*: proc(c: C): C {.nimcall.}
                
proc prim(s: string): C {.cpsMagic.} =
  echo s               
  return c          
                      
proc f(): C {.cps.} =
  var a: int = 0     
  for i in 0..3:    
    inc a, 1         
  assert a == 4        
  cps prim()           
                   
var c = f()                  
                     
while c != nil and c.fn != nil:
  c = c.fn(c)  

need some tests of generics

Incorrect rewrite for defer

import cps

type
  InfiniteLoop = CatchableError
  Cont* = ref object of RootObj
    when cpsMutant:
      fn*: proc(c: var Cont) {.nimcall.}
    else:
      fn*: proc(c: Cont): Cont {.nimcall.}

var jumps: int

proc trampoline(c: Cont) =
  jumps = 0
  var c = c
  while c != nil and c.fn != nil:
    c = c.fn(c)
    inc jumps
    if jumps > 1000:
      raise newException(InfiniteLoop, $jumps & " iterations")

proc noop*(c: Cont): Cont {.cpsMagic.} = c

template check(cond: bool, msg: string) =
  if not cond:
    raise newException(Defect, "not " & astToStr(cond) & ": " & msg)

var r = 0

proc foo() {.cps: Cont.} =
  defer:
    check r == 2, "defer run before end of scope"
    inc r

  inc r
  noop()
  inc r

trampoline foo()
check r == 3, "expected 3, got: " & $r

Current output:

Error: unhandled exception: not r == 2: defer run before end of scope [Defect]

`block:` not handled in transform

Generic continuations

What would be needed to get generic continuations like in stash/iteratorT.nim to work?

rewrite continue

taste test `noop` currently produces compiler assertion error

Error: unhandled exception: ccgexprs.nim(576, 9) `e[1].typ != nil`  [AssertionDefect]

for loops need special handling

move continuation return-type to macro (pragma syntax)

Getting rid of reallocs

Just writing this up because I'm still not sure if we're thinking the same thing.

How stuff works now

Some things I think are true:

• We have a proc that gets split up into a number of continuation procs.

• For each of these split procs, a dedicated environment object type is generated.

• An environment for a deeper nested scope inherits its parents environment.
  This is now done with Nim inheriting objects, but this could also be done by
  embedded parent environment objects as the first member of the child
  environment, or by making all these environments "from scratch" but taking
  care they for a proper tree hierarchy

Practical example:

The pre-transform original proc:

proc foo(a: int): Cont {.cps.} =
  echo "a = ", a
  var i: int = 0
  while i < 5:
    echo "i = ", i
    inc i
    inc a
  var j: int = 0
  while j < 5:
    echo "j = ", j
    inc j
    inc a
  echo "a = ", a

These are the generated environment types:

type
  env0 = ref object {.cpsLift.}of Cont
    a: int

type
  env1 = ref object {.cpsLift.}of env0
    j: int

type
  env2 = ref object {.cpsLift.}of env0
    i: int

In memory, these will look like this:

             ---
  @ 0x00:   | a | env0
             ---

           ------------
	  |  ---       |
  @ 0x00: | | a | env0 |
          |  ---       | env1
  @ 0x08: |   i        |
           ------------

           ------------
	  |  ---       |
  @ 0x00: | | a | env0 |
          |  ---       | env2
  @ 0x08: |   j        |
           ------------

Because i and j are never active at the same time but only in two
successive scopes, they will occupy the same location in env1 and env2.

This is what one of the split procs now looks like:

 1 proc brake(locals: Cont): Cont =
 2   let a: int = env0(locals).a
 3   block:
 4     block:
 5       var j: int = 0
 6       inc a
 7       return Cont(env1(fn: loop, j: j, a: a))
 8     return Cont(env0(fn: tail, a: a))

• At line 2 variables are pulled out of the environment into a local variable
• At line 6 and 7 a new environment for the next continuation is allocated, and the
  current locals are copied into the new environment

Problem statement

The problem we have now is that ever continuation gets a new, freshly allocated
environment. That's one alloc per trampoline call, which gets out of hand
quickly. Also, data get copied in and out environments for each call.

Aso, we can not simple rewrite local variables to environment access (i ->
env.i), since the (potentially modified) variables need to be copied into the
next environment just before the return.

Solution?

The solution would be to alloc one single block for the environment, and reuse
this block as it is passed between continuations. During collection of the
enviroment the maximum possible environment size can be calculated, which is
used as the initial alloc size.

• Don't copy locals out of the incoming environment, instead rewrite the code
  to use the vars from the env
• Convert/cast the passed env to the next type, update the members that need
  updating and pass that block to the next continuation

 1 proc brake(locals: Cont): Cont =
 2   block:
 3     block:
 4       var j: int = 0
 5       inc (env0(locals)).a
 6       result = cast[env1](locals)
 7       result.fn = loop
 8       result.j = j
 9       return result
10     result = env0(locals)
11     result.fn = tail

• Line 6 casts the incoming env to the next type 'env1'. We can not use a
  convert here since we might need to reuse the data for incompatible branches in
  the inheritance tree (where different members occupie the same address)
• Line 7 and 8 update the parts in the environment that need updating. Note that
  a is already in the right place as received from the incoming env

Optimizations

• Instead of allocating the maximum possible size of the env of this chain of continuations, we could start small and
  realloc only when needed.

support named break

break is broken

import cps 
 
type

  Cont = ref object of RootObj
    fn*: proc(c: Cont): Cont {.nimcall.}
    
    
proc test(): Cont {.cps.} = 
  
  while true:
    cps sleep()
    if true:
      break

brokenbreak.nim(10, 21) template/generic instantiation of `cps` from here                                 
brokenbreak.nim(15, 7) Error: invalid control flow: break  

Please prevent dependencies between CPS and this particular event queue implementation

Mornin' Disruptek,

Went through the code a bit, but I need to spend some more time to get into the details. One thing I'd like to propose: your current implementation kind of implies the use of the provided eventqueue which is used by all the tests and all, which also holds the bare essential thing to pump an continuation, the trampoline.

I'd like to propose to make a very explict split between the CPS part and the eventqueue part to make sure there will be no dependency creep of one single eventqueue/selector/whatever on the CPS implementation. We should be able to use and offer CPS building blocks with a pretty clean interface and put that on top of anything we want. So continuations know nothing of fds or timers or whatever is out there in the world, it's just the context itself.

transform bonkers

I might get to debug this myself, but not sure how much time I can spend today with the kids running around the house and the dishes piling up. For the record, just dumping it here:

I'm trying to get a minimal stand alone cps based server socket thing running. The snippet below will compile far enough to see the transformations with -d:cpsDebug, the whole thing is at http://ix.io/2shD

import "cps"                                                   
                     
type Cont = ref object of RootObj                             
  fn*: proc(c: Cont): Cont {.nimcall.} 
                                         
# CPS client hander
                                                       
proc doClient(fdc: SocketHandle): Cont {.cps.} =
                    
  while true:                                                                  
    cps io(fdc, POLLIN)                      
    let s: string = sockRecv(fdc)
    if s.len > 0:                          
      echo "recv> ", s                          
    else:                     
      return                                    
                                       
# CPS server handler                                                                  
                                          
proc doServer(port: int): Cont {.cps.} =
  let fds: SocketHandle = sockBind(port)          
  while true:            
    cps io(fds, POLLIN)
    let fdc: SocketHandle = sockAccept(fds)
    # Create new client and add to work queue
    evq.work.addLast doClient(fdc)

Some things derail for me now:

• The doClient loop gets transformed to this: note the double return

proc loop_14237113(locals_14237210: Cont): Cont =
  ## installing locals for env env
  let fdc: SocketHandle = env_14237084(locals_14237210).fdc
  if true:
    return io(env_14237084(fn: after_14237257, fdc: fdc).Cont, fdc, POLLIN)
    ## post-cps call; time to bail
    ## split at: after
    ## new tail call: after_14237257
    return env_14237084(fn: after_14237257, fdc: fdc).Cont
    ## creating a new proc: after_14237257

• Comping the whole thing from the ix.io snippet gives me C compiler errors:

/home/ico/.cache/nim/t_d/@mt.nim.c: In function ‘after__9bgcr1CRd8IRKJAnbwRHzMA_4’:
/home/ico/.cache/nim/t_d/@mt.nim.c:611:78: error: ‘tyObject_envcolonObjectType___baahae0jnkMVren5WI9bsYw’ has no member named ‘fdc’
  611 |  fdc = (*((tyObject_envcolonObjectType___baahae0jnkMVren5WI9bsYw*) (locals))).fdc;
      |                                                                              ^
/home/ico/.cache/nim/t_d/@mt.nim.c:638:14: error: ‘tyObject_envcolonObjectType___HggcRF6w2HNjc8xK1EHtaA’ has no member named ‘s’
  638 |  T8_ = (*T7_).s; (*T7_).s = copyStringRC1(s);
      |              ^
/home/ico/.cache/nim/t_d/@mt.nim.c:638:24: error: ‘tyObject_envcolonObjectType___HggcRF6w2HNjc8xK1EHtaA’ has no member named ‘s’
  638 |  T8_ = (*T7_).s; (*T7_).s = copyStringRC1(s);
      |                        ^
compilation terminated due to -fmax-errors=3.

iterative rewrite stages

• add missing returns
• normalize returns
• normalize proc params (3-son identDefs, at least)
• normalize var/let blocks (3-son identDefs, one section per identifier if possible)
• replace stack symbols with continuation symbols
• rewrite hidden conversions
• rewrite unhidden conversions
• replace hidden addr/deref
• normalize tuple deconstruction assignment
• try exprs
• defer
• rewrite iterators as cps
• rewrite cps calls as blocks
• rewrite cps while as blocks
• rewrite cps break as goto
• rewrite cps continue as goto
• rewrite cps blocks as procs
• rewrite named break as goto
• rewrite rvalue cps in shims
• remove gratuitous assignments? or omit them from earlier stage?
• lift types
• lift procs
• collapse types

Let's refine this list and add detail, stages...

add secret `result` in the continuation and retrieval method

Maybe ][? I know, it's nutty.

for loops could be supported with a compiler fix

Here's the test that runs against the almost branch:

https://github.com/disruptek/cps/blob/almost/tests/fors.nim

import testes
import cps

type
  C = ref object of RootObj
    fn*: proc(c: C): C {.nimcall.}

proc trampoline(c: C) =
  var c = c
  while c != nil and c.fn != nil:
    c = c.fn(c)

testes:

  block:
    proc loop() {.cps: C.} =
      for i in 0 .. 3:
        discard
    trampoline loop()

Here's the broken rewrite; CPS doesn't know what to do with the var i statement. I don't even understand how it passes through sem successfully to reach our macro...

 16  proc foo() =
 17    block :tmp:
 18      var i
 19      ## An alias for `countup(a, b, 1)`.
 20           ## 
 21           ## See also:
 22           ## * [..<](#..<.i,T,T)
 23      var res = 0
 24      block :tmp:
 25        while res <= 3:
 26          i = T(res)
 27          discard
 28          inc(res, 1)

expression 'exit_14240299' has no type (or is ambiguous)

import cps

type C = ref object of RootObj
  fn*: proc(c: C): C {.nimcall.}

proc add1(val: var int): C {.cpsMagic.} =
  inc val

proc f(): C {.cps.} =
  var a: int
  cps add1(a)
  if true:
    cps add1(a)
    if true:
      cps add1(a)
  doAssert a == 3

=== .cps. on f ===

type
  env_14240156 = ref object {.cpsLift.}of C
    a: int

proc after_14240296(locals_14240313: C): C =
  
proc after_14240307(locals_14240312: C): C =
  
proc after_14240307(locals_14240312: C): C =
  ## installing locals for env env
  var a: int = env_14240156(locals_14240312).a
  ## add if body
  if true:
    return add1(env_14240156(fn: exit_14240299, a: a).C, a)
    ## post-cps call; time to bail
    ## split at: after - no body left
    return env_14240156(fn: exit_14240299, a: a).C
  ## split at: maybe - no body left
  return env_14240156(fn: exit_14240299, a: a).C

proc exit_14240299(locals_14240300: C): C =
  
proc exit_14240299(locals_14240300: C): C =
  ## installing locals for env env
  var a: int = env_14240156(locals_14240300).a
  doAssert a == 3

proc after_14240296(locals_14240313: C): C =
  ## installing locals for env env
  var a: int = env_14240156(locals_14240313).a
  if true:
    return add1(env_14240156(fn: after_14240307, a: a).C, a)
    ## post-cps call; time to bail
    ## split at: after
    ## new tail call: after_14240307
    return env_14240156(fn: after_14240307, a: a).C
    ## creating a new proc: after_14240307
  ## add the exit proc definition
  ## split at: exit
  ## new tail call: exit_14240299
  return env_14240156(fn: exit_14240299, a: a).C
  ## creating a new proc: exit_14240299
  
proc f(): C =
  var a: int
  return add1(env_14240156(fn: after_14240296, a: a).C, a)
  ## post-cps call; time to bail
  ## split at: after
  ## new tail call: after_14240296
  return env_14240156(fn: after_14240296, a: a).C
  ## creating a new proc: after_14240296
  
/home/ico/sandbox/prjs/cps/t.nim(10, 15) template/generic instantiation of `cps` from here
/home/ico/sandbox/prjs/cps/cps.nim(368, 21) Error: expression 'exit_14240299' has no type (or is ambiguous)```

Splitting inside an if-else generates broken ast

import cps

type
  Cont* = ref object of RootObj
    fn*: proc(c: Cont): Cont {.nimcall.}

proc trampoline(c: Cont) =
  var c = c
  while c != nil and c.fn != nil:
    c = c.fn(c)

proc noop*(c: Cont): Cont {.cpsMagic.} = c

template check(b: bool, msg: string) =
  if not b:
    raise newException(AssertionDefect, msg)

var r = 0
proc b() {.cps: Cont.} =
  if true:
    noop()
    inc r
  else:
    discard
  inc r

trampoline b()
check r == 2, "expected 2, got: " & $r

Output:

=== .cps. on b(original)  === temp.nim(19, 0)
 19  proc b() =
 20    if true:
 21      noop()
 22      inc r, 1
 23    else:
 24      discard
 25    inc r, 1
 26
storing type env_402653455
next type env_402653738
=== .cps. on b(transform) === temp.nim(19, 0)
 19
 20  type
 21    env_402653455 = ref object of Cont
 22
 23  proc done_402653695(continuation: Cont): Cont {.cpsLift.}
 24  proc done_402653705(continuation: Cont): Cont {.cpsLift.}
 25  proc done_402653706(continuation: Cont): Cont {.cpsLift.}
 26  proc done_402653707(continuation: Cont): Cont {.cpsLift.}
 27  proc afterCall_402653708(continuation: Cont): Cont {.cpsLift.}
 28  proc done_402653695(continuation: Cont): Cont {.cpsLift.} =
 29    ## saften at 20 of temp.nim
 30    inc r, 1
 31
 32  proc done_402653705(continuation: Cont): Cont {.cpsLift.} =
 33    ## saften at 688 of cps.nim
 34    inc r, 1
 35
 36  proc done_402653706(continuation: Cont): Cont {.cpsLift.} =
 37    ## saften at 20 of temp.nim
 38    else:
 39      discard
 40
 41  proc done_402653707(continuation: Cont): Cont {.cpsLift.} =
 42    ## saften at 688 of cps.nim
 43    else:
 44      discard
 45
 46  proc afterCall_402653708(continuation: Cont): Cont {.cpsLift.} =
 47    ## saften at 871 of cps.nim
 48    inc r, 1
 49
 50  proc b(continuation: Cont): Cont {.cpsCall.} =
 51    ## saften at 19 of temp.nim
 52    ## saften at 20 of temp.nim
 53    ## saftening during done
 54    if true:
 55      ## saften at 21 of temp.nim
 56      ## re-use the local continuation by setting the fn
 57      continuation.fn = afterCall_402653708
 58      return noop(continuation)
 59      ## new tail call: done_402653707
 60      ## re-use the local continuation by setting the fn
 61      continuation.fn = done_402653707
 62      return continuation
 63    ## new tail call: done_402653705
 64    ## re-use the local continuation by setting the fn
 65    continuation.fn = done_402653705
 66    return continuation
 67
 68  proc b(): Cont =
 69    result = env_402653455(fn: b)

isolating a continuation doesn't work

See https://github.com/disruptek/cps/tree/threads

mutable params and the developers who love them

At some point I developed the conceit that this test should work. The question is whether @Clyybber and company are going to put the kibosh on this due to some hidden addr bs.

    block:
      ## reassignment of var proc params
      ## https://github.com/disruptek/cps/issues/22 (2nd)
      proc foo(a, b, c: var int) {.cps: Cont.} =
        a = 5 
        noop()    
        b = b + a
        noop()   
        check:  
          a == 5
          b == 7
          c == 3
      var (x, y, z) = (1, 2, 3)
      trampoline foo(x, y, z)
      check:
        x == 5
        y == 7
        z == 3

need a foreign module bindsym test

This is expected to fail, unfortunately.

What's all this stuff called anyway?

I'd like to start thinking about documenting some things, but don't know how to name things. Also, in all these discussion we sometimes seem to have a hard time naming the things we're taking about - but that could be only me of course.

Trying to assemble a list of CPS specific concepts below. I here call them by ugly, long, CAPITAL names, please suggest proper alternatives.

Actors/parties/humans

• CPS_DEVELOPER: the people making the CPS implementation. Know all the gory details of CPS. Disruptek.

• MIDDLEWARE_DEVELOPER: the people making MIDDLEWARE, i.e. cool things on top of cps. Need to know the outer API as offered by CPS, canb benefit by some level of insight of the CPS implementation. Zevv

• END_USER: Users of the MIDDLEWARE. These probably do not have to know about CPS, just as the do not have to know about the Nim compiler internals or Nim async implementation. We just provide the magic.

Code bases

• CPS: Original acronym of Continuation Passing Style, here used as the implementation of a framework for doing friendly CPS things in Nim. Also the disruptek/cps project, providing import cps.

• MIDDLEWARE: an implementation of some functionality built on top of cps. Things like async I/O, coroutines, iterators.

• USER_CODE: Code written by the END_USER making use of MIDDLEWARE. Does not need to import cps

CPS internal concepts

Things used by the CPS implementation, some of which are relevant for MIDDLEWARE.

• CPS_PROC: A proc that is written by the END_USER and gets transformed by the CPS_TRANSFORM into a number of CONTINUATION_PROCs. This is a proc that can have normal Nim program flow constructs, currently limited to if/then/else, while, break [label], continue and return.

• CPS_TRANSFORM: The process of splitting a CPS_PROC into a number of CONTINUATION_PROCS and doing LOCALS_LIFTING and BOXING, typically implemented by a Nim macro

• CONTINUATION_PROC: proc(c :T): T typically generated by the CPS_TRANSFORM from splitting up a CPS_PROC.

• CONTINUATION: a Nim concept describing a ref object with the members listed below. This is a real thing that lives at a place on the heap. The exact implementation of this has changed over versions and is still under development.

  • fn: a CONTINUATION_PROC pointer, which is the next proc to call for this continuation
  • Other MIDDLEWARE specific data, which is typically visible to CPS_MAGIC_PROCs but not to CPS_PROCs
  • The ENVIRONMENT for the current CONTINUATION_PROC. Might be one single ENVIRONMENT which holds locals for all CONTINUATION_PROCS that were derived from the same CPS_PROC

• LOCALS_LIFTING: The process of moving local (stack) variables from a CPS_PROC into the ENVIRONMENT, typically performed by the CPS_TRANSFORM

• CPS_MAGIC_PROC: implemented by the MIDDLEWARE to implement primitives for yielding, I/O await, and other fun things. This is a proc that has full access to the CONTINUATION and ENVIRONMENT

• ENVIRONMENT: a representation of a stack frame of an original CPS_PROC that now lives on the heap

Middleware concepts

• CONTINUE: call the fn of a CONTINUATION passing the CONTINUATION as an argument, and saving the result CONTINUATION for successive calls.

• FINISH: A CPS_MAGIC_PROC returning a nil CONTINUATION so the TRAMPOLINE can no longer CONTINUE the call chain. This can be happening when a process is truly "done", or when the call chain has been resumed for doing I/O wait or coroutine yielding. In this case the MIDDLEWARE will store the reference to the next CONTINUATION somewhere else and will resume CONTINUING at a later time.

• TRAMPOLINE: A function that CONTINUES a continuation until it FINISHES

• EVENT_QUEUE: A typical MIDDLEWARE implementation that provides primitives like async I/O. It might need bookkeeping like a ioselector, containers holding CONTINUATIONs that will be CONTINUED when I/O is ready.

Bah this ended up much longer then I expected to. Sorry for the noise

need a test of exceptions

Transformation does not do proper .{cpsCall.}

import cps

type
  C = ref object of RootObj
    fn: proc(c: C): C

proc jield(c: C): C {.cpsMagic.} =
  discard                         
                                  
proc count() {.cps:C.} =
  var i = 0             
  while i < 2:
    jield()                  
               
var c = count()
c = c.fn(c)       
c = c.fn(c)

Resuts in the compile time error:

/home/ico/sandbox/prjs/cps/stash/minimal_example.nim(17) minimal_example
/home/ico/sandbox/prjs/cps/stash/minimal_example.nim(13) whileLoop
/home/ico/sandbox/prjs/cps/stash/minimal_example.nim(7) jield
Error: unhandled exception: jield() is only valid in {.cps.} context [Defect]

cpsDebug shows that jield() is not properly handled as a cps call:

850  proc count(continuation: C): C {.cpsCall().} =
851    ## saften at 10 of minimal_example.nim
852    ## saften at 11 of minimal_example.nim
853    ## saften at 11 of minimal_example.nim
854    env_402653446(continuation).i_402653654 = 0
855    proc whileLoop_402653792(continuation: C): C {.cpsLift().} =
856      if env_402653446(continuation).i_402653654 < 2:
857        jield()
858      else:
859        continuation.fn = nil
860        return continuation
861      continuation.fn = whileLoop_402653792
862      return continuation

Bonus points for explaining to me why this happens: If the last line of the program c = c.fn(c) is dropped. compilation succeeds, even though the rewrite is the same.

shadowed variables don't work due to name clashes

double returns generated

need some tests of traces

expression 'brake_16985017' has no type (or is ambiguous)

Related to #10?

import cps      
                 
type C = ref object of RootObj
  fn*: proc(c: C): C {.nimcall.}
                 
proc add1(val: var int): C {.cpsMagic.} =
  inc val            

proc f(): C {.cps.} =
      var a: int
      cps add1(a)
      block:     
        cps add1(a)
        break      
        cps add1(a)  
      doAssert a == 2

=== .cps. on t ===

type
  env_16985002 = ref object {.cpsLift.}of C
    a: int

proc after_16985016(locals_16985029: C): C =
  
proc after_16985023(locals_16985026: C): C =
  
proc after_16985023(locals_16985026: C): C =
  ## installing locals for env env
  var a: int = env_16985002(locals_16985026).a
  ## simple break statement
  return env_16985002(fn: brake_16985017, a: a).C
  return add1(env_16985002(fn: nil, a: a).C, a)
  ## post-cps call; time to bail
  ## split at: after - no body left
  return nil

proc tail_16985027(locals_16985028: C): C =
  
proc brake_16985017(locals_16985018: C): C =
  
proc brake_16985017(locals_16985018: C): C =
  ## installing locals for env env
  var a: int = env_16985002(locals_16985018).a
  doAssert a == 2

proc tail_16985027(locals_16985028: C): C =
  ## installing locals for env env
  var a: int = env_16985002(locals_16985028).a
  ## split at: brake
  ## new tail call: brake_16985017
  return env_16985002(fn: brake_16985017, a: a).C
  ## creating a new proc: brake_16985017
  
proc after_16985016(locals_16985029: C): C =
  ## installing locals for env env
  var a: int = env_16985002(locals_16985029).a
  block:
    return add1(env_16985002(fn: after_16985023, a: a).C, a)
    ## post-cps call; time to bail
    ## split at: after
    ## new tail call: after_16985023
    return env_16985002(fn: after_16985023, a: a).C
    ## creating a new proc: after_16985023
  ## add tail call for block-break proc
  ## new tail call: tail_16985027
  return env_16985002(fn: tail_16985027, a: a).C
  ## creating a new proc: tail_16985027
  
proc t(): C =
  var a: int
  return add1(env_16985002(fn: after_16985016, a: a).C, a)
  ## post-cps call; time to bail
  ## split at: after
  ## new tail call: after_16985016
  return env_16985002(fn: after_16985016, a: a).C
  ## creating a new proc: after_16985016
  
/home/ico/sandbox/prjs/cps/tests/tzevv.nim(32, 7) template/generic instantiation of `suite` from here
/home/ico/sandbox/prjs/cps/tests/tzevv.nim(138, 8) template/generic instantiation of `test` from here
/home/ico/sandbox/prjs/cps/tests/tzevv.nim(139, 5) template/generic instantiation of `runCps` from here
/home/ico/sandbox/prjs/cps/tests/tzevv.nim(23, 17) template/generic instantiation of `cps` from here
/home/ico/sandbox/prjs/cps/cps.nim(368, 21) Error: expression 'brake_16985017' has no type (or is ambiguous)

wrong transformation, premature return

proc doServer(port: int) {.cps:Cont} =
  let fds: SocketHandle = sockBind(port)
  while true:                                
    io(fds, 1.int16)                                    
    echo "accepting"                                           
    let fdc: SocketHandle = sockAccept(fds)       
    echo "accepted"           

expected result: when connecting to the port, print "accepting", accept the socket and print "accepted"
actual result: print "accepting", accept the socket, and premature return from there

The above gets translated into:

126  proc afterCall_369102403(continuation: Cont): Cont {.cpsLift.} =
127    ## saften at 848 of cps.nim
128    echo "accepting"
129    ## saften at 113 of standalone_tcp_server.nim
130    env_369102398(continuation).fdc_369102402 = sockAccept(env_369102398(continuation).fds_369102400)
132    ## adding return call to nnkStmtList
133    ## re-use the local continuation by setting the fn
134    continuation.fn = whileLoop_369102401
135    return continuation
136    echo "accepted"
137    addLast(evq.work, doClient(env_369102398(continuation).fdc_369102402))
138    ## saften at 511 of cps.nim
139    ## re-use the local continuation by setting the fn
140    continuation.fn = whileLoop_369102401
141    return continuation
142    ## omit return call from nnkStmtList

Line 134..135 wrongly inserts a return after the sockAccept()

multiple variable declaration does not work

with var i,j: int j is ignored