mkholodnyak / node-search-engine-parser

Search Engine Parser

Node.js - Search Engine Parser

This module allows you to search google by scraping the results. You can add your own search engine.

Installation

npm install --save search-engine-parser

Built-in strategies

• Google Images

Usage

This prints out the first 20 search results of the query kitten in Google Images.

var createParser = require('search-engine-parser');

var googleImagesParser = createParser('google-images');

googleImagesParser.search('kitten', function(err, results){
    results.map(function(result){
        console.log(result);
    });
});

Own search engine

You need to implement Search Strategy interface:

var cheerio = require('cheerio');

var SearchEngineStrategy = {
   getSearchURL : function(query){
       return "http://your-search-engine.com?query=" + query;
   },

   getResults : function(html, callback){
       var $ = cheerio.load(html);
       var imageLinks = $('.link');
       var imageHrefs = [];

       imageLinks.each(function(i, element){
           imageHrefs.push(element.attribs.href);
       });

       callback(null, imageHrefs);
   }
};

Example Google Images strategy.

License

Licensed under MIT.

node-search-engine-parser's Issues

Google Search Strategy

Hi,

I have noticed that the provided strategy returns an array of #'s. I poked around a little. I think what Google provides now is rendered as a bunch of a and img tags whose attributes match the regexes by the browser but what is actually provided when requested from node.js is div's with JSON strings inside. So I modified it and it now returns as expected.

var cheerio = require('cheerio');

var IMAGE_LINKS_SELECTOR = 'div.rg_meta.notranslate'; // JSON strings are in these div's
var GOOGLE_SEARCH_URL = 'http://images.google.com/search?tbm=isch&q=';

var GoogleImageStrategy = {
    getSearchURL : function(query){
        return GOOGLE_SEARCH_URL + query
    },

    getResults : function(html, callback){
        var $ = cheerio.load(html);
        var imageLinks = $(IMAGE_LINKS_SELECTOR);
        var imageHrefs = [];

        imageLinks.each(function collectHref(i, element){
            imageHrefs.push($(element).text()); // Need to get JSON strings inside the div's
        });
        
        callback(null, imageHrefs);
    }
};

module.exports = GoogleImageStrategy;