File Upload about pywinrm

[d0c-s4vage]

I got binary file transfers to work just fine with something like this:

        def put_file(self, location, contents):
                step = 400
                for i in range(0, len(contents), step):
                        self._do_put_file(location, contents[i:i+step])

        def _do_put_file(self, location, contents):
                # adapted/copied from https://github.com/diyan/pywinrm/issues/18
                ps_script = """
$filePath = "{location}"
$s = @"
{b64_contents}
"@
$data = [System.Convert]::FromBase64String($s)
add-content -value $data -encoding byte -path $filePath
                """.format(
                        location                = location,
                        b64_contents    = base64.b64encode(contents)
                )

                r = self._sess.run_ps(ps_script)
                if r.status_code == 1:
                        self._log.warn(r.std_err)
                        return None

                self._log.debug(r.std_out)
                return r.std_out

[pdunnigan]

I am pretty sure that winrm does not support file transfer regardless of what client library you use. It uses an http soap protocol so you can't stream files. There is no ftp-like interface.

That being said, there is a way to use powershell to write files. I am not sure how it would handle binary files though and may be subject to the winrm server settings (concerning size of script you can send to the server). However this works well for writing text based, multiline config files. see below, comments in the sample script.

from winrm import Protocol
import base64
address = "10.10.0.12"
transport = "plaintext"
username = "administrator"
password = "ThePassword"
protocol = "http"
port = 5985

endpoint = "%s://%s:%s/wsman" % (protocol, address, port)

conn = Protocol(endpoint=endpoint, transport=transport, username=username, password=password)
shell_id = conn.open_shell()

# the text file we want to send
# this could be populated by reading a file from disk instead
# has some special characters, just to prove they won't be a problem
text_file = """this is a multiline file
that contains special characters such as
"blah"
'#@$*&&($}
that will be written
onto the windows box"""

# first part of the powershell script
# streamwriter is the fastest and most efficient way to write a file
# I have found
# notice the @", this is like a "here document" in linux
# or like triple quotes in python and allows for multiline files
part_1 = """$stream = [System.IO.StreamWriter] "test.txt"
$s = @"
"""

# second part of the powershell script
# the "@ closes the string
# the replace will change the linux line feed to the windows carriage return, line feed
part_2 = """
"@ | %{ $_.Replace("`n","`r`n") }
$stream.WriteLine($s)
$stream.close()"""

# join the beginning of the powershell script with the text file and end of the ps script
script = part_1 + text_file + part_2

# base64 encode, utf16 little endian. required for windows
encoded_script = base64.b64encode(script.encode("utf_16_le"))

# send the script to powershell, tell it the script is encoded
command_id = conn.run_command(shell_id, "powershell -encodedcommand %s" % (encoded_script))
stdout, stderr, return_code = conn.get_command_output(shell_id, command_id)
conn.cleanup_command(shell_id, command_id)
print "STDOUT: %s" % (stdout)
print "STDERR: %s" % (stderr)

# print the file
command_id = conn.run_command(shell_id, "type test.txt")
stdout, stderr, return_code = conn.get_command_output(shell_id, command_id)
conn.cleanup_command(shell_id, command_id)
print "STDOUT: %s" % (stdout)
print "STDERR: %s" % (stderr)

# delete the file
command_id = conn.run_command(shell_id, "del test.txt")
stdout, stderr, return_code = conn.get_command_output(shell_id, command_id)
conn.cleanup_command(shell_id, command_id)
print "STDOUT: %s" % (stdout)
print "STDERR: %s" % (stderr)

# always good to clean things up, doesn't hurt
conn.close_shell(shell_id)

[vipulob]

Wow. Thanks. I didn't know something like this can also be done.
I tried above example it worked fine.
I will close this ticket.

[thorsummoner]

Is this information still correct? I see many ruby/go packages that suppose to support file transport over winrm.

[pdunnigan]

@thorsummoner - point us to examples of ruby / go that have file transfer, I'd like to take a look.

[d0c-s4vage]

Sorry, really not meaning to butt in @thorsummoner and @pdunnigan , but I happen to be working on this at this very moment and figured I'd pitch in. My example above seems to work pretty well for binary files. The main ruby winrm client that I know of is https://github.com/WinRb/WinRM. Some file system utilities for it exist at https://github.com/WinRb/winrm-fs.

Their file upload mechanism is similar (not using official WinRm specs, but base64 encode and append to a file in chunks):

(from https://github.com/WinRb/winrm-fs/blob/master/lib/winrm-fs/core/file_uploader.rb)

        def prepare_script(remote_file)
          <<-EOH
            $p = $ExecutionContext.SessionState.Path
            $path = $p.GetUnresolvedProviderPathFromPSPath("#{remote_file}")
            # if the file exists, truncate it
            if (Test-Path $path -PathType Leaf) {
              '' | Set-Content $path
            }
            # ensure the target directory exists
            $dir = [System.IO.Path]::GetDirectoryName($path)
            if (!(Test-Path $dir -PathType Container)) {
              mkdir $dir -ErrorAction SilentlyContinue | Out-Null
            }
          EOH
        end

makes sure the directory exists before uploading, and

        def do_upload(local_file, remote_file)
          bytes_copied = 0
          base64_array = base64_content(local_file)
          base64_array.each_slice(8000 - remote_file.size) do |chunk|
            @command_executor.run_cmd("echo #{chunk.join} >> \"#{remote_file}\"")
            bytes_copied += chunk.count
            yield bytes_copied, base64_array.count if block_given?
          end
          base64_array.length
        end

uploads the file in chunks

[thorsummoner]

@d0c-s4vage Do you have a fork of this repo that implements your patch? Also do you do file integrity checks, and are you going to implement download capabilities too? I would be interested in contributing to python based efforts to archive this functionality 👍 Just link me to your project and i'll make a thread for us

[d0c-s4vage]

@thorsummoner no, I do not have a fork that implements it. Hopefully I'll get back to doing some dev on the project that was using this in the next week or so and can make a fork and submit a pull request. The code I have above is essentially what it would come down to, although I would want to do more testing to see if there are more efficient methods, larger chunk sizes, etc.

[thorsummoner]

Cool, I don't mind waiting, (targeting windows systems is the last thing I
want to do anyhow)
On Jun 30, 2015 7:36 AM, "James Johnson" [email protected] wrote:

@thorsummoner https://github.com/thorsummoner no, I do not have a fork
that implements it. Hopefully I'll get back to doing some dev on the
project that was using this in the next week or so and can make a fork and
submit a pull request. The code I have above is essentially what it would
come down to, although I would want to do more testing to see if there are
more efficient methods, larger chunk sizes, etc.

—
Reply to this email directly or view it on GitHub
#18 (comment).

[kgriffs]

FWIW, this was implemented in Ansible's winrm module, but note the warning. I verified these symptoms in my own tests:

https://github.com/ansible/ansible-modules-core/blob/devel/windows/win_copy.py#L50

[vipulob]

@d0c-s4vage:
The code is too slow for MB size files.

[Blackbaud-JohnHamill]

Have tried using the sample code above, works great for small files, or the example. Anything longer than about 12kb seems to cause "command line too long" errors with PowerShell. I believe there is a limit on how big a file PowerShell will accept on the command line for the encodedcommand.

[Blackbaud-JohnHamill]

The win_copyfile module is actually a custom action plug in, as well as the .ps1 in the core repo. It isn't using the module in the ansible-modules-core repo for the actual linux to windows transfer. I haven't done an analysis to see where the slowness is coming from, but a cursory look at the code shows its doing a lot of stuff. Its apparently doing a remote CRC of the file and somewhat trying to optimize copy - only copy if necessary etc. and check results. It also copies over the win_stat and win_copy modules on every transfer to do the copy from temp to the actual desired folder, so there is a lot of overhead. Win_copy is a good for a file or two, I wouldn't recommend copying a folder full of files unless you like to wait, or its a scheduled task and waiting doesn't really matter.

[sankamuk]

I am having issue running the code from @pdunnigan its giving the below issue.

Cannot process the command because the value specified with -EncodedCommand is not properly encoded. The value must be Base64 encoded.

Can there be any help on this. Thanks.

[mcassaniti]

If you've copied and pasted that code in full it should work. If it does not then try changing the line

# base64 encode, utf16 little endian. required for windows
encoded_script = base64.b64encode(script.encode("utf_16_le"))

to

# base64 encode, utf16 little endian. required for windows
encoded_script = base64.b64encode(script.encode('utf_16_le')).decode('ascii')

It's an untested suggestion and stolen from the winrm.Session.run_ps code.

[sankamuk]

@mcassaniti thanks so much. Solved for me.

[utkonos]

To update @d0c-s4vage's snippet to use some of the latest Python niceness and a tqdm progress meter for good measure.

import base64
import pathlib

import tqdm.auto
import winrm

s = winrm.Session('192.0.2.1', auth=('Administrator', 'password'))

test_file = pathlib.Path('test.dat')

write_chunk = """
[Byte[]]$to_write = [System.Convert]::FromBase64String('{}')
$to_write | Add-Content -Encoding Byte C:\{}
"""

chunk_size = 2048

with tqdm.auto.tqdm(total=test_file.stat().st_size) as t, with open(test_file, 'rb') as fh:
    while chunk := fh.read(chunk_size):
        encoded = base64.standard_b64encode(chunk)
        command = write_chunk.format(encoded.decode(), test_file.name)
        r = s.run_ps(command)
        if r.status_code:
            break
        t.update(chunk_size)

upload_file.mp4

Here is all that in a function:

import hashlib
import base64
import pathlib

import tqdm.auto
import winrm

def send_file(s, source, destination):
    """Send file to remote location in base64 encoded chunks via WinRM."""
    chunk_size = 2048
    write_chunk = "[Byte[]]$to_write = [System.Convert]::FromBase64String('{}')\n$to_write | Add-Content -Encoding Byte {}"

    with tqdm.auto.tqdm(total=source.stat().st_size) as t, with source.open('rb') as fh:
        while chunk := fh.read(chunk_size):
            encoded = base64.standard_b64encode(chunk).decode()
            command = write_chunk.format(encoded, destination)
            result = s.run_ps(command)
            if result.status_code:
                break
            t.update(chunk_size)

    if 'Completed' not in result.std_err.decode():
        raise RuntimeError(f'File send failed: {source.name}')

    get_sha256 = 'Get-FileHash {} -Algorithm SHA256 | ConvertTo-Json'
    command = get_sha256.format(destination)
    result = s.run_ps(command)
    if result.status_code:
        raise RuntimeError(f'Remote hash calculation failed: {source.name}')

    data = json.loads(result.std_out)
    sha256_remote = data.get('Hash').lower()

    file_data = source.read_bytes()
    sha256_local = hashlib.sha256(file_data).hexdigest().lower()
    if sha256_remote != sha256_local:
        raise RuntimeError(f'Hash mismatch: {source.name}')

s = winrm.Session('192.0.2.1', auth=('Administrator', 'password'))
source = pathlib.Path('test.dat')
destination = f'C:\{source.name}'

send_file(s, source, destination)