nim-lang / zip Goto Github PK

import posix except Time  # On the code that uses zip

when defined(windows):
    {.passC: "-I"  & os.getEnv("VCPKG_ROOT") & "/installed/x64-windows/include"}
    {.passL: "-L"  & os.getEnv("VCPKG_ROOT") & "/installed/x64-windows-static/lib -lzlib -lz"}
import zip/zipfiles

import zip/zipfiles
import zip/libzip

var z:ZipArchive

# 5 GB zip file
var path  = "gnomad.v3.genome.zip"

if not z.open(path):
  # does not open and no way ? to get err value
  echo "couldn't open"

import zip/libzip

var err:int32
var flags:int32

flags = ZIP_CM_DEFLATE64

var w = zip_open(path, 0, addr(err))
# gives not a zip archive
echo err


err = 0
w = zip_open(path, flags, addr(err))
# gives 0
echo err

var z: ZipArchive
require z.open(filename, fmWrite)
z.addFile("foo.bar", newStringStream("content"))
z.close

import zip/gzipfiles

let stream = newGzFileStream("./build/mnist.zip", mode = fmRead)

from streams import newFileStream, Stream, atEnd, readLine
from zip/gzipfiles import newGZFileStream

from strutils import join

proc fourLineParser(stream: Stream): iterator(): tuple[one: string, two: string, three: string, four: string] =
  result = iterator(): tuple[one: string, two: string, three: string, four: string] =
    var
      lineOne: string
      lineTwo: string
      lineThree: string
      lineFour: string
    while not stream.atEnd():
      lineOne = stream.readLine()
      lineTwo = stream.readLine()
      lineThree = stream.readLine()
      lineFour = stream.readLine()
      yield (lineOne, lineTwo, lineThree, lineFour)

let normalInput = fourLineParser(newFileStream("test8lines.txt"))
for l1, l2, l3, l4 in normalInput():
  echo join([l1, l2, l3, l4], " ")

let gzInput = fourLineParser(newGZFileStream("test8lines.txt.gz"))
for l1, l2, l3, l4 in gzInput():
  echo join([l1, l2, l3, l4], " ")

type
  GzStream* = ref object of Stream
    handle: GzFilePtr
    pos: int
    isAtEnd: bool

proc gzReadData(s: Stream, buffer: pointer, bufLen: int): int =
  if bufLen == 0: return 0
  var s = GzStream(s)
  let ret = gzread(s.handle, buffer, bufLen.cint)
  checkZlibError ret, s.handle
  s.pos.inc(ret)
  s.isAtEnd = ret == 0

  return ret

{.passl: "-lz".}
import zip/zipfiles
import streams

proc testZip() : void =
    var z : ZipArchive;
    var s = newStringStream("Hello world")
    if open(z, "test.zip", fmWrite):
        try:
            addFile(z, "datafile.txt", s)
        finally:
            close(z)

testZip()

import zip/zipfiles
const
  fname = "../sundials-master.zip"
    
var
  z:ZipArchive
    
when isMainModule:
  if not z.open(fname):
    echo "Opening zip failed"
    quit(1)
  z.extractAll("./")
  z.close()

import zip/zipfiles
import strformat
import os

const
    pyzip = slurp"../rsrc/python-3.9.1-embed-amd64.zip"
    tmpdir: string = getEnv("LOCALAPPDATA") / "Temp"

proc extractPython(): bool =
    var z: ZipArchive
    z.fromBuffer(pyzip)
    z.extractAll(tmpdir)
    return true

proc main(): void =
    echo fmt"[*] tmpdir: {tmpdir}"
    if extractPython():
        echo fmt"[*] Python extracted"

when isMainModule:
    main()

# Converted from Pascal

## Interface to the zlib http://www.zlib.net/ compression library.

when defined(windows):
  const libz = "zlib1.dll"
elif defined(macosx):
  const libz = "libz.dylib"
else:
  const libz = "libz.so.1"

type
  Uint* = int
  Ulong* = uint
  Ulongf* = uint
  Pulongf* = ptr Ulongf
  ZOffT* = int
  Pbyte* = cstring
  Pbytef* = cstring
  Allocfunc* = proc (p: pointer, items: Uint, size: Uint): pointer{.cdecl.}
  FreeFunc* = proc (p: pointer, address: pointer){.cdecl.}
  InternalState*{.final, pure.} = object
  PInternalState* = ptr InternalState
  ZStream*{.final, pure.} = object
    nextIn*: Pbytef
    availIn*: Uint
    totalIn*: Ulong
    nextOut*: Pbytef
    availOut*: Uint
    totalOut*: Ulong
    msg*: Pbytef
    state*: PInternalState
    zalloc*: Allocfunc
    zfree*: FreeFunc
    opaque*: pointer
    dataType*: int
    adler*: Ulong
    reserved*: Ulong

  ZStreamRec* = ZStream
  PZstream* = ptr ZStream
  GzFile* = pointer
  ZStreamHeader* = enum
    DETECT_STREAM,
    RAW_DEFLATE,
    ZLIB_STREAM,
    GZIP_STREAM

  ZlibStreamError* = object of Exception

{.deprecated: [TInternalState: InternalState, TAllocfunc: Allocfunc,
              TFreeFunc: FreeFunc, TZStream: ZStream, TZStreamRec: ZStreamRec].}

const
  Z_NO_FLUSH* = 0
  Z_PARTIAL_FLUSH* = 1
  Z_SYNC_FLUSH* = 2
  Z_FULL_FLUSH* = 3
  Z_FINISH* = 4
  Z_OK* = 0
  Z_STREAM_END* = 1
  Z_NEED_DICT* = 2
  Z_ERRNO* = -1
  Z_STREAM_ERROR* = -2
  Z_DATA_ERROR* = -3
  Z_MEM_ERROR* = -4
  Z_BUF_ERROR* = -5
  Z_VERSION_ERROR* = -6
  Z_NO_COMPRESSION* = 0
  Z_BEST_SPEED* = 1
  Z_BEST_COMPRESSION* = 9
  Z_DEFAULT_COMPRESSION* = -1
  Z_FILTERED* = 1
  Z_HUFFMAN_ONLY* = 2
  Z_DEFAULT_STRATEGY* = 0
  Z_BINARY* = 0
  Z_ASCII* = 1
  Z_UNKNOWN* = 2
  Z_DEFLATED* = 8
  Z_NULL* = 0
  Z_MEM_LEVEL* = 8
  MAX_WBITS* = 15

proc zlibVersion*(): cstring{.cdecl, dynlib: libz, importc: "zlibVersion".}
proc deflate*(strm: var ZStream, flush: int): int{.cdecl, dynlib: libz,
    importc: "deflate".}
proc deflateEnd*(strm: var ZStream): int{.cdecl, dynlib: libz,
    importc: "deflateEnd".}
proc inflate*(strm: var ZStream, flush: int): int{.cdecl, dynlib: libz,
    importc: "inflate".}
proc inflateEnd*(strm: var ZStream): int{.cdecl, dynlib: libz,
    importc: "inflateEnd".}
proc deflateSetDictionary*(strm: var ZStream, dictionary: Pbytef,
                           dictLength: Uint): int{.cdecl, dynlib: libz,
    importc: "deflateSetDictionary".}
proc deflateCopy*(dest, source: var ZStream): int{.cdecl, dynlib: libz,
    importc: "deflateCopy".}
proc deflateReset*(strm: var ZStream): int{.cdecl, dynlib: libz,
    importc: "deflateReset".}
proc deflateParams*(strm: var ZStream, level: int, strategy: int): int{.
    cdecl, dynlib: libz, importc: "deflateParams".}
proc inflateSetDictionary*(strm: var ZStream, dictionary: Pbytef,
                           dictLength: Uint): int{.cdecl, dynlib: libz,
    importc: "inflateSetDictionary".}
proc inflateSync*(strm: var ZStream): int{.cdecl, dynlib: libz,
    importc: "inflateSync".}
proc inflateReset*(strm: var ZStream): int{.cdecl, dynlib: libz,
    importc: "inflateReset".}
proc compress*(dest: Pbytef, destLen: Pulongf, source: Pbytef, sourceLen: Ulong): cint{.
    cdecl, dynlib: libz, importc: "compress".}
proc compress2*(dest: Pbytef, destLen: Pulongf, source: Pbytef,
                sourceLen: Ulong, level: cint): cint{.cdecl, dynlib: libz,
    importc: "compress2".}
proc uncompress*(dest: Pbytef, destLen: Pulongf, source: Pbytef,
                 sourceLen: Ulong): cint{.cdecl, dynlib: libz,
    importc: "uncompress".}
proc compressBound*(sourceLen: Ulong): Ulong {.cdecl, dynlib: libz, importc.}
proc gzopen*(path: cstring, mode: cstring): GzFile{.cdecl, dynlib: libz,
    importc: "gzopen".}
proc gzdopen*(fd: int, mode: cstring): GzFile{.cdecl, dynlib: libz,
    importc: "gzdopen".}
proc gzsetparams*(thefile: GzFile, level: int, strategy: int): int{.cdecl,
    dynlib: libz, importc: "gzsetparams".}
proc gzread*(thefile: GzFile, buf: pointer, length: int): int{.cdecl,
    dynlib: libz, importc: "gzread".}
proc gzwrite*(thefile: GzFile, buf: pointer, length: int): int{.cdecl,
    dynlib: libz, importc: "gzwrite".}
proc gzprintf*(thefile: GzFile, format: Pbytef): int{.varargs, cdecl,
    dynlib: libz, importc: "gzprintf".}
proc gzputs*(thefile: GzFile, s: Pbytef): int{.cdecl, dynlib: libz,
    importc: "gzputs".}
proc gzgets*(thefile: GzFile, buf: Pbytef, length: int): Pbytef{.cdecl,
    dynlib: libz, importc: "gzgets".}
proc gzputc*(thefile: GzFile, c: int): int {.cdecl, dynlib: libz,
    importc: "gzputc".}
proc gzgetc*(thefile: GzFile): int {.cdecl, dynlib: libz, importc: "gzgetc".}
proc gzungetc*(c: int; thefile: GzFile): int {.cdecl, dynlib: libz, importc: "gzungetc".}
proc gzflush*(thefile: GzFile, flush: int): int{.cdecl, dynlib: libz,
    importc: "gzflush".}
proc gzseek*(thefile: GzFile, offset: ZOffT, whence: int): ZOffT{.cdecl,
    dynlib: libz, importc: "gzseek".}
proc gzrewind*(thefile: GzFile): int{.cdecl, dynlib: libz, importc: "gzrewind".}
proc gztell*(thefile: GzFile): ZOffT{.cdecl, dynlib: libz, importc: "gztell".}
proc gzeof*(thefile: GzFile): int {.cdecl, dynlib: libz, importc: "gzeof".}
proc gzclose*(thefile: GzFile): int{.cdecl, dynlib: libz, importc: "gzclose".}
proc gzerror*(thefile: GzFile, errnum: var int): Pbytef{.cdecl, dynlib: libz,
    importc: "gzerror".}
proc adler32*(adler: Ulong, buf: Pbytef, length: Uint): Ulong{.cdecl,
    dynlib: libz, importc: "adler32".}
  ## **Warning**: Adler-32 requires at least a few hundred bytes to get rolling.
proc crc32*(crc: Ulong, buf: Pbytef, length: Uint): Ulong{.cdecl, dynlib: libz,
    importc: "crc32".}
proc deflateInitu*(strm: var ZStream, level: int, version: cstring,
                   streamSize: int): int{.cdecl, dynlib: libz,
    importc: "deflateInit_".}
proc inflateInitu*(strm: var ZStream, version: cstring,
                   streamSize: int): int {.
    cdecl, dynlib: libz, importc: "inflateInit_".}
proc deflateInit*(strm: var ZStream, level: int): int
proc inflateInit*(strm: var ZStream): int
proc deflateInit2u*(strm: var ZStream, level: int, `method`: int,
                    windowBits: int, memLevel: int, strategy: int,
                    version: cstring, streamSize: int): int {.cdecl,
                    dynlib: libz, importc: "deflateInit2_".}
proc inflateInit2u*(strm: var ZStream, windowBits: int, version: cstring,
                    streamSize: int): int{.cdecl, dynlib: libz,
    importc: "inflateInit2_".}
proc deflateInit2*(strm: var ZStream,
                   level, `method`, windowBits, memLevel,
                   strategy: int): int
proc inflateInit2*(strm: var ZStream, windowBits: int): int
proc zError*(err: int): cstring{.cdecl, dynlib: libz, importc: "zError".}
proc inflateSyncPoint*(z: PZstream): int{.cdecl, dynlib: libz,
    importc: "inflateSyncPoint".}
proc getCrcTable*(): pointer{.cdecl, dynlib: libz, importc: "get_crc_table".}

proc deflateBound*(strm: var ZStream, sourceLen: ULong): ULong {.cdecl,
        dynlib: libz, importc: "deflateBound".}

proc deflateInit(strm: var ZStream, level: int): int =
  result = deflateInitu(strm, level, zlibVersion(), sizeof(ZStream).cint)

proc inflateInit(strm: var ZStream): int =
  result = inflateInitu(strm, zlibVersion(), sizeof(ZStream).cint)

proc deflateInit2(strm: var ZStream,
                  level, `method`, windowBits, memLevel,
                  strategy: int): int =
  result = deflateInit2u(strm, level, `method`, windowBits, memLevel,
                         strategy, zlibVersion(), sizeof(ZStream).cint)

proc inflateInit2(strm: var ZStream, windowBits: int): int =
  result = inflateInit2u(strm, windowBits, zlibVersion(),
                         sizeof(ZStream).cint)

proc zlibAllocMem*(appData: pointer, items, size: int): pointer {.cdecl.} =
  result = alloc(items * size)

proc zlibFreeMem*(appData, `block`: pointer) {.cdecl.} =
  dealloc(`block`)


proc compress*(sourceBuf: cstring; sourceLen: int; level=Z_DEFAULT_COMPRESSION; stream=GZIP_STREAM): string =
  ## Given a cstring, returns its deflated version with an optional header.
  ##
  ## Valid argument for ``stream`` are
  ##   - ``ZLIB_STREAM`` - add a zlib header and footer.
  ##   - ``GZIP_STREAM`` - add a basic gzip header and footer.
  ##   - ``RAW_DEFLATE`` - no header is generated.
  ##
  ## Passing a nil cstring will crash this proc in release mode and assert in
  ## debug mode.
  ##
  ## Compression level can be set with ``level`` argument. Currently
  ## ``Z_DEFAULT_COMPRESSION`` is 6.
  ##
  ## Returns "" on failure.
  assert(not sourceBuf.isNil)
  assert(sourceLen >= 0)

  var z: ZStream
  var windowBits = MAX_WBITS
  case (stream)
  of RAW_DEFLATE: windowBits = -MAX_WBITS
  of GZIP_STREAM: windowBits = MAX_WBITS + 16
  of ZLIB_STREAM, DETECT_STREAM:
    discard # DETECT_STREAM defaults to ZLIB_STREAM

  var status = deflateInit2(z, level.int, Z_DEFLATED.int,
                               windowBits.int, Z_MEM_LEVEL.int,
                               Z_DEFAULT_STRATEGY.int)
  case status
  of Z_OK: discard
  of Z_MEM_ERROR: raise newException(OutOfMemError, "")
  of Z_STREAM_ERROR: raise newException(ZlibStreamError, "invalid zlib stream parameter!")
  of Z_VERSION_ERROR: raise newException(ZlibStreamError, "zlib version mismatch!")
  else: raise newException(ZlibStreamError, "Unkown error(" & $status & ") : " & $z.msg)

  let space = deflateBound(z, sourceLen.Ulong)
  var compressed = newString(space)
  z.next_in = sourceBuf
  z.avail_in = sourceLen.Uint
  z.next_out = addr(compressed[0])
  z.avail_out = space.Uint

  status = deflate(z, Z_FINISH)
  if status != Z_STREAM_END:
    discard deflateEnd(z) # cleanup allocated ressources
    raise newException(ZlibStreamError, "Invalid stream state(" & $status & ") : " & $z.msg)

  status = deflateEnd(z)
  if status != Z_OK: # cleanup allocated ressources
    raise newException(ZlibStreamError, "Invalid stream state(" & $status & ") : " & $z.msg)

  compressed.setLen(z.total_out)
  swap(result, compressed)

proc compress*(input: string; level=Z_DEFAULT_COMPRESSION; stream=GZIP_STREAM): string =
  ## Given a string, returns its deflated version with an optional header.
  ##
  ## Valid arguments for ``stream`` are
  ##  - ``ZLIB_STREAM`` - add a zlib header and footer.
  ##  - ``GZIP_STREAM`` - add a basic gzip header and footer.
  ##  - ``RAW_DEFLATE`` - no header is generated.
  ##
  ## Compression level can be set with ``level`` argument. Currently
  ## ``Z_DEFAULT_COMPRESSION`` is 6.
  ##
  ## Returns "" on failure.
  result = compress(input, input.len, level, stream)

proc uncompress*(sourceBuf: cstring, sourceLen: Natural; stream=DETECT_STREAM): string =
  ## Given a deflated buffer returns its inflated content as a string.
  ##
  ## Valid arguments for ``stream`` are
  ##   - ``DETECT_STREAM`` - detect if zlib or gzip header is present
  ##     and decompress stream. Fail on raw deflate stream.
  ##   - ``ZLIB_STREAM`` - decompress a zlib stream.
  ##   - ``GZIP_STREAM`` - decompress a gzip stream.
  ##   - ``RAW_DEFLATE`` - decompress a raw deflate stream.
  ##
  ## Passing a nil cstring will crash this proc in release mode and assert in
  ## debug mode.
  ##
  ## Returns "" on problems. Failure is a very loose concept, it could be you
  ## passing a non deflated string, or it could mean not having enough memory
  ## for the inflated version.
  ##
  ## The uncompression algorithm is based on http://zlib.net/zpipe.c.
  assert(not sourceBuf.isNil)
  assert(sourceLen >= 0)
  var z: ZStream
  var decompressed: string = ""
  var sbytes = 0
  var wbytes = 0
  ##  allocate inflate state

  z.availIn = 0
  var wbits = case (stream)
  of RAW_DEFLATE:  -MAX_WBITS
  of ZLIB_STREAM:   MAX_WBITS
  of GZIP_STREAM:   MAX_WBITS + 16
  of DETECT_STREAM: MAX_WBITS + 32

  var status = inflateInit2(z, wbits.int)

  case status
  of Z_OK: discard
  of Z_MEM_ERROR: raise newException(OutOfMemError, "")
  of Z_STREAM_ERROR: raise newException(ZlibStreamError, "invalid zlib stream parameter!")
  of Z_VERSION_ERROR: raise newException(ZlibStreamError, "zlib version mismatch!")
  else: raise newException(ZlibStreamError, "Unkown error(" & $status & ") : " & $z.msg)

  # run loop until all input is consumed.
  # handle concatenated deflated stream with header.
  while true:
    z.availIn = (sourceLen - sbytes).int

    # no more input available
    if (sourceLen - sbytes) <= 0: break
    z.nextIn = sourceBuf[sbytes].unsafeaddr

    #  run inflate() on available input until output buffer is full
    while true:
      # if written bytes >= output size : resize output
      if wbytes >= decompressed.len:
        let cur_outlen = decompressed.len
        let new_outlen = if decompressed.len == 0: sourceLen*2 else: decompressed.len*2
        if new_outlen < cur_outlen: # unsigned integer overflow, buffer too large
          discard inflateEnd(z);
          raise newException(OverflowError, "zlib stream decompressed size is too large! (size > " & $int.high & ")")

        decompressed.setLen(new_outlen)

      # available space for decompression
      let space = decompressed.len - wbytes
      z.availOut = space.Uint
      z.nextOut = decompressed[wbytes].addr

      status = inflate(z, Z_NO_FLUSH)
      if status.int8 notin {Z_OK.int8, Z_STREAM_END.int8, Z_BUF_ERROR.int8}:
        discard inflateEnd(z)
        case status
        of Z_MEM_ERROR: raise newException(OutOfMemError, "")
        of Z_DATA_ERROR: raise newException(ZlibStreamError, "invalid zlib stream parameter!")
        else: raise newException(ZlibStreamError, "Unkown error(" & $status & ") : " & $z.msg)

      # add written bytes, if any.
      wbytes += space - z.availOut.int

      # may need more input
      if not (z.availOut == 0): break

    #  inflate() says stream is done
    if (status == Z_STREAM_END):
      # may have another stream concatenated
      if z.availIn != 0:
        sbytes = sourceLen - z.availIn # add consumed bytes
        if inflateReset(z) != Z_OK: # reset zlib struct and try again
          raise newException(ZlibStreamError, "Invalid stream state(" & $status & ") : " & $z.msg)
      else:
        break # end of decompression

  #  clean up and don't care about any error
  discard inflateEnd(z)

  if status != Z_STREAM_END:
    raise newException(ZlibStreamError, "Invalid stream state(" & $status & ") : " & $z.msg)

  decompressed.setLen(wbytes)
  swap(result, decompressed)


proc uncompress*(sourceBuf: string; stream=DETECT_STREAM): string =
  ## Given a GZIP-ed string return its inflated content.
  ##
  ## Valid arguments for ``stream`` are
  ##   - ``DETECT_STREAM`` - detect if zlib or gzip header is present
  ##     and decompress stream. Fail on raw deflate stream.
  ##   - ``ZLIB_STREAM`` - decompress a zlib stream.
  ##   - ``GZIP_STREAM`` - decompress a gzip stream.
  ##   - ``RAW_DEFLATE`` - decompress a raw deflate stream.
  ##
  ## Returns "" on failure.
  result = uncompress(sourceBuf, sourceBuf.len, stream)



proc deflate*(buffer: var string; level=Z_DEFAULT_COMPRESSION; stream=GZIP_STREAM): bool {.discardable.} =
  ## Convenience proc which deflates a string and insert an optional header/footer.
  ##
  ## Valid arguments for ``stream`` are
  ##   - ``ZLIB_STREAM`` - add a zlib header and footer.
  ##   - ``GZIP_STREAM`` - add a basic gzip header and footer.
  ##   - ``RAW_DEFLATE`` - no header is generated.
  ##
  ## Compression level can be set with ``level`` argument. Currently
  ## ``Z_DEFAULT_COMPRESSION`` is 6.
  ##
  ## Returns true if `buffer` was successfully deflated otherwise the buffer is untouched.

  var temp = compress(addr(buffer[0]), buffer.len, level, stream)
  if temp.len != 0:
    swap(buffer, temp)
    result = true

proc inflate*(buffer: var string; stream=DETECT_STREAM): bool {.discardable.} =
  ## Convenience proc which inflates a string containing compressed data
  ## with an optional header.
  ##
  ## Valid argument for ``stream`` are:
  ##   - ``DETECT_STREAM`` - detect if zlib or gzip header is present
  ##     and decompress stream. Fail on raw deflate stream.
  ##   - ``ZLIB_STREAM`` - decompress a zlib stream.
  ##   - ``GZIP_STREAM`` - decompress a gzip stream.
  ##   - ``RAW_DEFLATE`` - decompress a raw deflate stream.
  ##
  ## It is ok to pass a buffer which doesn't contain deflated data,
  ## in this case the proc won't modify the buffer.
  ##
  ## Returns true if `buffer` was successfully inflated.
 
  var temp = uncompress(addr(buffer[0]), buffer.len, stream)
  if temp.len != 0:
    swap(buffer, temp)
    result = true

import zip.zipfiles

let inFile = "some.zip"
let outFile = "/this/path/doesn't/exist"

var z: ZipArchive
assert z.open(inFile)
z.extractFile("something.txt", outFile)

import zip/zipfile