GithubHelp home page GithubHelp logo

ans-4175 / indonesian-news-scraper Goto Github PK

View Code? Open in Web Editor NEW

This project forked from anpandu/indonesian-news-scraper

0.0 2.0 0.0 634 KB

Indonesian Online News Scraper

License: MIT License

JavaScript 100.00%

indonesian-news-scraper's Introduction

indonesian-news-scraper

NPM version Build Status Dependency Status

NodeJS module for scraping Indonesian news site. Websites available:

  • Kompas
  • Detik
  • Tempo
  • KapanLagi
  • TempoEnglish
  • Antara
  • Republika
  • Okezone
  • Liputan6
  • Viva

Install

$ npm install --save indonesian-news-scraper

Usage

Scrap

Scrap all news in the website's main page. The number of news scraped varies according to website.

var kompas = require('indonesian-news-scraper').Kompas;

kompas.scrap()
  .then(function (scraps) {
    console.log(scraps[0])
  })

/***
{ url: 'http://megapolitan.kompas.com/read/2015/11/05/13554811/DKI.Akan.Tambah.Dana.Hibah.untuk.Wilayah.yang.Dilintasi.Truk.Sampah',
  title: 'DKI Akan Tambah Dana Hibah untuk Wilayah yang Dilintasi Truk Sampah',
  date: '2015-11-05 T10:08:55Z',
  img: 'http://assets.kompas.com/data/photo/2015/05/14/173048120150507-140842780x390.JPG',
  content: 'Sekretaris Daerah (Sekda) DKI Saefullah ..... ',
  source: 'Kompas' }
***/

kompas.scrap()
  .then(function (scraps) {
    var titles = scraps.map(function (news) {return news.title})
    console.log(titles)
  })

/***
[ 'DKI Akan Tambah Dana Hibah untuk Wilayah yang Dilintasi Truk Sampah',
  'Sanusi Anggap Sikap Ahok Picu Aksi Warga Tolak Truk Sampah DKI',
  'Persiapan Daerah Menghadapi MEA',
  'Tangani Kasus Kebakaran Lahan, Kejagung Bantu Kejaksaan di Daerah',
  '"Tak Perlu Khawatir dengan Letusan Gunung Rinjani"',
  'Ini 5 Menteri Terbaik Jokowi-JK Versi Survei LSJ',
  'Warga Siap Demo Lebih Besar jika Truk DKI Langgar Kesepakatan',
  'Kumpul di Istana, Ini yang Dibahas Jokowi dan Pimpinan DPR',
  'Yusril Ajak DKI Selesaikan Masalah Sampah dengan Damai',
  'Kisruh Sampah DKI, Polisi Minta Pemprov dan Pengelola Lakukan Mediasi',
  'Dipecat, Buruh Bawa Anak Istri Mengadu ke DPRD',
  'Bahas Beberapa Kasus Korupsi, Kabareskrim Temui Pimpinan KPK' ]
***/

Get URLs

Get all URLs in the website's main page

var kompas = require('indonesian-news-scraper').Kompas;

kompas.getURLs()
  .then(function (urls) {
    console.log(urls)
  })

/***
[ 'http://megapolitan.kompas.com/read/2015/11/09/12372421/Wakil.Wali.Kota.Bekasi.Izin.Melintas.24.Jam.Truk.Sampah.DKI.Hanya.Sementara',
  'http://regional.kompas.com/read/2015/11/09/12305981/Terduga.Teroris.yang.Tewas.di.Sulteng.Belum.Teridentifikasi',
  'http://megapolitan.kompas.com/read/2015/11/09/12103131/Bicarakan.Kisruh.Sampah.Ahok.Akan.Bertemu.Wali.Kota.Bekasi.',
  'http://nasional.kompas.com/read/2015/11/09/11555121/Masa.Peminjaman.Gedung.BNN.Milik.Polri.Habis.Desember.2015',
  'http://megapolitan.kompas.com/read/2015/11/09/11543021/Truk.Sampah.DKI.Diizinkan.Melintas.24.Jam.DPRD.Bekasi.Tetap.Ingin.Evaluasi.MoU.',
  'http://nasional.kompas.com/read/2015/11/09/11434001/Sore.Ini.Presiden.Italia.Akan.Temui.Jokowi.di.Istana.Merdeka',
  'http://megapolitan.kompas.com/read/2015/11/09/11425941/Kampung.Pulo.Masih.Kebanjiran.Warga.Minta.Ahok.Buat.Gorong-gorong',
  'http://megapolitan.kompas.com/read/2015/11/09/11395131/Organda.Jangan.Selalu.Salahkan.Sopir.',
  'http://internasional.kompas.com/read/2015/11/09/11372961/Kerusuhan.Pecah.di.Pusat.Tahanan.Imigrasi.Australia.di.Christmas.Island',
  'http://nasional.kompas.com/read/2015/11/09/11363951/Fahri.Hamzah.Sebut.Soeharto.Layak.Dapat.Gelar.Pahlawan',
  'http://nasional.kompas.com/read/2015/11/09/11354671/Kasus.Suap.APBD.KPK.Panggil.9.Anggota.DPRD.Sumut.2009-2014',
  'http://regional.kompas.com/read/2015/11/09/11294151/Antisipasi.Ancaman.Teroris.Petugas.Keamanan.Khusus.Disiagakan.di.Candi' ]
***/

Available Scrappers

var kompas = require('indonesian-news-scraper').Kompas;
var detik = require('indonesian-news-scraper').Detik;
var tempo = require('indonesian-news-scraper').Tempo;
var kapanLagi = require('indonesian-news-scraper').KapanLagi;
var tempoEnglish = require('indonesian-news-scraper').TempoEnglish;
var antara = require('indonesian-news-scraper').Antara;
var republika = require('indonesian-news-scraper').Republika;
var okezone = require('indonesian-news-scraper').Okezone;
var liputan6 = require('indonesian-news-scraper').Liputan6;
var viva = require('indonesian-news-scraper').Viva;

License

MIT © Ananta Pandu

indonesian-news-scraper's People

Contributors

anpandu avatar

Watchers

 avatar  avatar

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. 📊📈🎉

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google ❤️ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.