Implement screen capture, save, open, and search. about robotjs

[Zazck]

@arfost process image as buffer(not as array) will speed up your convertion.

function screenCaptureToFile(robotScreenPic) {
  return new Promise((resolve, reject) => {
    try {
      const image = new Jimp(robotScreenPic.width, robotScreenPic.height);
      let pos = 0;
      image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
        /* eslint-disable no-plusplus */
        image.bitmap.data[idx + 2] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 1] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 0] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 3] = robotScreenPic.image.readUInt8(pos++);
        /* eslint-enable no-plusplus */
      });
      resolve(image);
    } catch (e) {
      console.error(e);
      reject(e);
    }
  });
}

it will be 400 times faster

buffer: 50 convert: 21213
buffer: 55 convert: 21277
buffer: 40 convert: 21823

[TongHaizhang]

Maybe you can speed up the pace of updates

[carlosgalveias]

While saving the screen capture is not implemented, here's a workaround i used by using the Jimp package:
https://github.com/oliver-moran/jimp

var Jimp = require('jimp');
var robot = require('robotjs');

var screenCaptureToFile = function(path) {
    return new Promise((resolve, reject) => {
        try {
            var picture = robot.screen.capture();
            var image = new Jimp(picture.width, picture.height, function(err, img) {
                img.bitmap.data = picture.image;
                img.getBuffer(Jimp.MIME_PNG, (err, png) => {
                    image.write(path, resolve);
                });
            });
        } catch (e) {
            console.error(e);
            reject(e);
        }
    })
}

screenCaptureToFile('test.png');

[omegacoleman]

The solution @carlosgalveias given has made two mistakes, the correct version might be:

    var image = jimp(pic.width, pic.height, function(err, img) {
        img.bitmap.data = pic.image;
        img.scan(0, 0, img.bitmap.width, img.bitmap.height, function (x, y, idx) {
          var red   = img.bitmap.data[ idx + 0 ];
          var blue  = img.bitmap.data[ idx + 2 ];
          img.bitmap.data[ idx + 0 ] = blue;
          img.bitmap.data[ idx + 2 ] = red;
        });
        img.write("./" + n + (new Date().getTime()) + '.png');
    });

but this need improving as well. Jimp and RobotJS pick differently from BGR and RGB, so it needs reversing. Actually I think performing an option in robotjs about RGB and BGR is a better idea.

[will123195]

This worked for me:

const robot = require('robotjs')
const Jimp = require('jimp')

function captureImage({ x, y, w, h }) {
  const pic = robot.screen.capture(x, y, w, h)
  const width = pic.byteWidth / pic.bytesPerPixel // pic.width is sometimes wrong!
  const height = pic.height
  const image = new Jimp(width, height)
  let red, green, blue
  pic.image.forEach((byte, i) => {
    switch (i % 4) {
      case 0: return blue = byte
      case 1: return green = byte
      case 2: return red = byte
      case 3: 
        image.bitmap.data[i - 3] = red
        image.bitmap.data[i - 2] = green
        image.bitmap.data[i - 1] = blue
        image.bitmap.data[i] = 255
    }
  })
  return image
}

captureImage({ x, y, w, h }).write('capture.png')

[carlosgalveias]

Ok, i managed to do it in C++ and now its fast.

Made a fork here https://github.com/carlosgalveias/robotjs in windows that outputs the buffer as a RGB and not as a BRG

Added the function to MMBitmap.c :

MMBitmapRef ConvertBMPToRGBBuffer (MMBitmapRef bitmap) {

assert(bitmap != NULL);
uint8_t *rgbBitmap = NULL;

if (bitmap->imageBuffer != NULL) {
	const size_t bufsize = bitmap->height * bitmap->bytewidth;
	rgbBitmap = malloc(bufsize);
	if (rgbBitmap == NULL) return NULL;
	memcpy(rgbBitmap, bitmap->imageBuffer, bufsize);
}

// find the number of padding bytes
int padding = 0;
int scanlinebytes = bitmap->bytewidth;
while ( ( scanlinebytes + padding ) % bitmap->bytewidth != 0 )     // DWORD = 4 bytes
	padding++;
// get the padded scanline bitmap->bytewidth
int psw = scanlinebytes + padding;

// now we loop trough all bytes of the original buffer, 
// swap the R and B bytes and the scanlines
long bufpos = 0;   
long newpos = 0;
for ( int y = 0; y < bitmap->height; y++ )
	for ( int x = 0; x < bitmap->bytesPerPixel * bitmap->width; x+=bitmap->bytesPerPixel )
	{
		newpos = y * bitmap->bytesPerPixel * bitmap->width + x;     
		rgbBitmap[newpos] = bitmap->imageBuffer[newpos + 2];       
		rgbBitmap[newpos + 2] = bitmap->imageBuffer[newpos];     
	}

// Do not forget to destroy the original bitmap if you dont need it to avoid memory leaks
return createMMBitmap(rgbBitmap,
                      bitmap->width,
                      bitmap->height,
                      bitmap->bytewidth,
                      bitmap->bitsPerPixel,
                      bitmap->bytesPerPixel); }

On screengrab.c i added:

/* Copy the data to our pixel buffer. */

if (bitmap != NULL) {
	bitmap->imageBuffer = malloc(bitmap->bytewidth * bitmap->height);
	memcpy(bitmap->imageBuffer, data, bitmap->bytewidth * bitmap->height);
	bitmapRGB = ConvertBMPToRGBBuffer(bitmap);
}
destroyMMBitmap(bitmap);

thanks @Zazck for that snippet.

I was having an issue on my computer with the alpha channel not rendering correctly; it was spitting out large black rectangles over certain areas. In case anyone else is having the same issue, this forces the alpha channel to always be 100% during the conversion:

function screenCaptureToFile(robotScreenPic) {
  return new Promise((resolve, reject) => {
    try {
      const image = new Jimp(robotScreenPic.width, robotScreenPic.height);
      let pos = 0;
      image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
        /* eslint-disable no-plusplus */
        image.bitmap.data[idx + 2] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 1] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 0] = robotScreenPic.image.readUInt8(pos++);
        pos++
        image.bitmap.data[idx + 3] = 255;
        /* eslint-enable no-plusplus */
      });
      resolve(image);
    } catch (e) {
      console.error(e);
      reject(e);
    }
  });
}

[porsager]

@octalmage Also curious if this is being worked on or if you'd be ok with a pull request? I'd probably be interested in making the call async. What do you think about that?

[arfost]

Hi,

I'm not sure why but the two above functions with jimp didn't work for me, and I needed to save the captured picture, so here is my version of the function using JIMP if it can help someone :)

var Jimp = require('jimp');
var robot = require('robotjs');

var screenCaptureToFile = function (path,  robotScreenPic) {
    return new Promise((resolve, reject) => {
        try {
            var image = new Jimp(robotScreenPic.width, robotScreenPic.height);
            image.scan(0, 0, image.bitmap.width, image.bitmap.height, function (x, y, idx) {
                var color = robotScreenPic.colorAt(x, y);
                var red = parseInt(color[0] + color[1], 16);
                var green = parseInt(color[2] + color[3], 16);
                var blue = parseInt(color[4] + color[5], 16);

                image.bitmap.data[idx + 0] = Number(red);
                image.bitmap.data[idx + 1] = Number(green);
                image.bitmap.data[idx + 2] = Number(blue);
                image.bitmap.data[idx + 3] = 255;
            });
            image.write(path, resolve);

        } catch (e) {
            console.error(e);
            reject(e);
        }
    })
}

Thanks to @carlosgalveias and @omegacoleman for the starter, I couldn't find it without them.
Beware, it can be very slow for big pictures

[octalmage]

For the bitmap functions, I'll return an object similar to this:

MMBitmapRef createMMBitmap(uint8_t *buffer,
                           size_t width,
                           size_t height,
                           size_t bytewidth,
                           uint8_t bitsPerPixel,
                           uint8_t bytesPerPixel)
{
    MMBitmapRef bitmap = malloc(sizeof(MMBitmap));
    if (bitmap == NULL) return NULL;

    bitmap->imageBuffer = buffer;
    bitmap->width = width;
    bitmap->height = height;
    bitmap->bytewidth = bytewidth;
    bitmap->bitsPerPixel = bitsPerPixel;
    bitmap->bytesPerPixel = bytesPerPixel;

    return bitmap;
}

This way we can pass the necessary

[octalmage]

With the new syntax this will go in the index.js.

function bitmap (imagedata, width, height) 
{
    this.imagedata = imagedata;
    this.width = width;
    this.height. = height;
}

bitmap.prototype.save = function(path) 
{
    robotjs.saveBitmap(this.imagedata, path);
};

module.exports.screen.capture = function()
{
    b = robotjs.captureScreen();
    return new bitmap(b.imagedata, b.width, b.height);
}

Then you'll use it like this:

var robot = require("robotjs");

var desktop = robot.screen.capture();

desktop.save("screencapture.png");

Note: I wrote this on my phone.

[octalmage]

Might need to use:

createStringFromMMBitmap
createMMBitmapFromString

To pass the image between V8 and JavaScript.

[Deltatiger]

This should be the next priority. Capture screen will allow for more projects to adopt RobotJS. (Also my mini project of a remote desktop )

[octalmage]

Agreed, I would like to implement this since there's a special way I want it implemented (see bitmap branch). I'll start on this after #87 is merged and RobotJS is completely updated. This is going to take some extra work on Windows due to the libpng and zlib dependencies, which is the main reason I put it off. These libraries are required for disk related functionality, and createMMBitmapFromString.

#64 is another priority for me.

[octalmage]

Here's the latest:

http://blog.robotjs.io/post/130642552013/i-spent-a-few-hours-yesterday-and-today-working-on

[Deltatiger]

@octalmage I will look into this bug and get it sorted out. Is the version implemented for Linux ?

[octalmage]

Nice! Yeah it works for Linux too. I'm pretty sure my math is wrong for the buffers though. If I remember correctly it crashed when I passed the image buffer back to C++ from JavaScript in colorAt.

[SiriusDG]

Is the image save implemented yet? I tried to follow the demo from comments, desktop.save did not yet work. Any simple way to save captured images? This is about all I really need to make great use of this, thanks!

[octalmage]

The first part is implemented, screen capturing, but not saving yet. We just released the capture code in 0.4.0 this weekend, so it will be a bit longer before the saving code is out.

[roccomuso]

+1

[stephenthedev]

@octalmage is there an update on this? Has the ready tag since 10/15

[Hum4n01d]

Any news?

[happilymarrieddad]

+1

[octalmage]

@porsager I've got screen capture implemented, but that's it. There's logic in there for the rest but it needs some work. See https://github.com/octalmage/robotjs/tree/bitmap-save.

I don't have the time to work on this currently (and won't for a while), but I'm happy to accept and review PRs.

Thanks!

[developer239]

Thank you so much.

[johnmarlow007]

Hi guys. I just wanted to say "CONGRATULATIONS!" for this project.
I have tried a few things and it works pretty much flowlessly.

Now I too need this screen capture feature with image saved locally for further processing and I highly encourage your efforts to get it working.

I would like to support the project but will need to get deeper in the study of how its done and I hope I can help with my limited knowledge.
Thank you and best of luck !
Andy

[vernondegoede]

@arfost I experienced the same issue with large pictures (especially retina screens). It took too long to process the picture (about ~600ms), so I ended up using the desktop-screenshot package.

https://github.com/vernondegoede/nonbilight/blob/master/renderer/components/TheaterMode.js#L50

[carlosgalveias]

In the meantime i found this:
http://tipsandtricks.runicsoft.com/Cpp/BitmapTutorial.html#chapter4

Maybe this can be done natively in C++ so that its faster, in certain cases the JIMP solution is taking 2 seconds.
But getting the output buffer as a RGB instead of a BRG is a must for me.
I would implement it, and tried but crashes all over the place (i'm not a c++ programmer)

@octalmage , what do you think ?

[Deltatiger]

This has to be done in C++ to get some speed benefits. Currently the entire screen buffer's memory is copied onto a bitmap object in C++ and then returned to JS. The code snippet posted above iterates over each pixel (I guess) before the image is generated. For a simple 1024 x 768 resolution screen (Which is a very small screen compared to what is actually out there) this is going to run a lot of times.
Instead if RobotJS binary can directly create the specified file in BMP format, it should be way easier and a lot quicker.
And since the data is still being generated as a Bitmap image, creating a BMP file should be easy.

[jtoy]

is this considered complete?

[Sarkell]

@fanfare I didn't come across this issue. And I have been using this solution for a week very intensively

@octalmage It would be wonderful to have this feature ASAP. The solution seems to be found

[sebastiaorealino]

This worked for me:

const robot = require('robotjs')
const Jimp = require('jimp')

function captureImage({ x, y, w, h }) {
  const pic = robot.screen.capture(x, y, w, h)
  const width = pic.byteWidth / pic.bytesPerPixel // pic.width is sometimes wrong!
  const height = pic.height
  const image = new Jimp(width, height)
  let red, green, blue
  pic.image.forEach((byte, i) => {
    switch (i % 4) {
      case 0: return blue = byte
      case 1: return green = byte
      case 2: return red = byte
      case 3: 
        image.bitmap.data[i - 3] = red
        image.bitmap.data[i - 2] = green
        image.bitmap.data[i - 1] = blue
        image.bitmap.data[i] = 255
    }
  })
  return image
}

captureImage({ x, y, w, h }).write('capture.png')

Try to use @Zazck solution.
His code is almost twice as fast:

@will123195 code: 639.065ms
@Zazck code: 371.819ms

Thanks @Zazck!

[wangrongding]

期待

[wburgess-around]

If anyone else just needs to grab a screenshot on MacOS you can just do:

await robot.keyTap('3', [ 'command', 'shift' ])

to take a screenshot that will auto-save to your Desktop. Obviously will not support custom sizes but this is all I needed to debug a remote machine.

[Psalm01-bot]

Weldone at octalmage, pls can you pls update on search function

[arthurwolf]

Any progress on this?

Amazing project.