Building a News Aggregator: Web Scraping Tutorial

const response = await fetch('https://api.robotscraping.com/extract', {
  method: 'POST',
  headers: {
    'Content-Type': 'application/json',
    'x-api-key': process.env.API_KEY
  },
  body: JSON.stringify({
    url: 'https://news.example.com/technology',
    fields: ['headlines'],
    instructions: 'Extract all news article headlines from this page. For each headline, return the title, the article URL, the publication date if shown, a brief summary or excerpt, and the category or section name. Return as an array of objects.'
  })
});

const data = await response.json();
// {
//   headlines: [
//     {
//       title: "Tech Giant Announces New AI Features",
//       url: "https://news.example.com/article1",
//       date: "January 15, 2025",
//       summary: "The company revealed...",
//       category: "Technology"
//     },
//     {
//       title: "Startup Raises $50M Series B",
//       url: "https://news.example.com/article2",
//       date: "January 15, 2025",
//       summary: "The funding will be used...",
//       category: "Business"
//     },
//     // ... more headlines
//   ]
// }

const response = await fetch('https://api.robotscraping.com/extract', {
  method: 'POST',
  headers: {
    'Content-Type': 'application/json',
    'x-api-key': process.env.API_KEY
  },
  body: JSON.stringify({
    url: 'https://news.example.com/article/tech-giant-ai',
    fields: [
      'article_title',
      'author',
      'publication_date',
      'last_updated',
      'article_body',
      'summary',
      'tags',
      'category',
      'image_url',
      'image_caption'
    ],
    instructions: 'Extract the main article content. Get the headline, byline with author name, publication and update dates, the full article text body (not including ads or sidebars), a brief summary, any topic tags or categories, and the main image with its caption.'
  })
});

const article = await response.json();
// {
//   article_title: "Tech Giant Announces New AI Features",
//   author: "Jane Smith",
//   publication_date: "2025-01-15",
//   last_updated: "2025-01-15 14:30",
//   article_body: "In a groundbreaking announcement...",
//   summary: "The company revealed new AI capabilities...",
//   tags: ["AI", "Technology", "Product Launch"],
//   category: "Technology",
//   image_url: "https://news.example.com/images/article1.jpg",
//   image_caption: "The new AI interface in action."
// }

const articleUrls = [
  'https://news.example.com/article1',
  'https://news.example.com/article2',
  'https://news.example.com/article3'
];

const response = await fetch('https://api.robotscraping.com/batch', {
  method: 'POST',
  headers: {
    'Content-Type': 'application/json',
    'x-api-key': process.env.API_KEY
  },
  body: JSON.stringify({
    urls: articleUrls,
    fields: ['article_title', 'article_body', 'author', 'publication_date'],
    async: true,
    webhook_url: 'https://your-app.com/webhook/articles'
  })
});

-- News sources configuration
CREATE TABLE news_sources (
  id INTEGER PRIMARY KEY,
  name TEXT NOT NULL,
  base_url TEXT NOT NULL,
  category_pages TEXT, -- JSON array of category URLs
  rss_feeds TEXT, -- JSON array of RSS feed URLs
  update_frequency INTEGER, -- Minutes between checks
  is_active BOOLEAN,
  last_crawled_at TIMESTAMP
);

-- Extracted articles
CREATE TABLE articles (
  id INTEGER PRIMARY KEY,
  source_id INTEGER,
  url TEXT UNIQUE,
  title TEXT,
  author TEXT,
  publication_date TIMESTAMP,
  article_body TEXT,
  summary TEXT,
  category TEXT,
  tags TEXT, -- JSON array
  image_url TEXT,
  scraped_at TIMESTAMP,
  FOREIGN KEY (source_id) REFERENCES news_sources(id)
);

-- Article deduplication tracking
CREATE TABLE article_clusters (
  id INTEGER PRIMARY KEY,
  canonical_article_id INTEGER,
  cluster_signature TEXT,
  article_ids TEXT, -- JSON array of related article IDs
  created_at TIMESTAMP
);

// Get sources that need to be crawled
async function getSourcesForCrawling(db) {
  const sources = await db.prepare(`
    SELECT * FROM news_sources
    WHERE is_active = true
      AND (
        last_crawled_at IS NULL
        OR datetime(last_crawled_at, '+' || update_frequency || ' minutes') < datetime('now')
      )
  `).all();
  return sources;
}

// Mark source as crawled
async function markSourceCrawled(db, sourceId) {
  await db.prepare(`
    UPDATE news_sources
    SET last_crawled_at = datetime('now')
    WHERE id = ?
  `).bind(sourceId).run();
}

// Store extracted articles
async function storeArticles(db, sourceId, articles) {
  for (const article of articles) {
    await db.prepare(`
      INSERT OR REPLACE INTO articles
      (source_id, url, title, author, publication_date,
       article_body, summary, category, tags, image_url, scraped_at)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `).bind(
      sourceId,
      article.url,
      article.title,
      article.author,
      article.publication_date,
      article.article_body,
      article.summary,
      article.category,
      JSON.stringify(article.tags || []),
      article.image_url,
      Date.now()
    ).run();
  }
}

// Simple URL normalization
function normalizeUrl(url) {
  return url
    .toLowerCase()
    .replace(/^https?:///, '')
    .replace(//$/, '')
    .replace(/www./, '')
    .replace(/#.*/, '');
}

// Check for duplicates before inserting
async function findDuplicateByUrl(db, url) {
  const normalized = normalizeUrl(url);
  return await db.prepare(`
    SELECT id FROM articles WHERE normalize_url(url) = ?
  `).bind(normalized).first();
}

// Generate a simple content signature
function generateContentSignature(title, summary) {
  const words = (title + ' ' + summary)
    .toLowerCase()
    .replace(/[^ws]/g, '')
    .split(/s+/)
    .filter(w => w.length > 3)
    .sort();

  // Create hash from sorted unique words
  return crypto.createHash('md5')
    .update(words.join(' '))
    .digest('hex');
}

// Find similar articles
async function findSimilarArticles(db, title, summary) {
  const signature = generateContentSignature(title, summary);
  return await db.prepare(`
    SELECT a.*, c.signature
    FROM articles a
    LEFT JOIN article_clusters c ON a.id = c.canonical_article_id
    WHERE c.signature = ?
  `).bind(signature).all();
}

app.post('/webhook/news/:source', async (req, res) => {
  const { source } = req.params;
  const { result } = req.body;

  // Verify webhook signature
  const signature = req.headers['x-webhook-signature'];
  if (!verifySignature(signature, req.body, WEBHOOK_SECRET)) {
    return res.status(401).send('Invalid signature');
  }

  // Get source ID
  const sourceRecord = await getSourceByName(source);
  if (!sourceRecord) {
    return res.status(404).send('Source not found');
  }

  // Process headlines
  const { headlines } = result;
  for (const headline of headlines) {
    // Check for duplicates
    const existing = await findDuplicateByUrl(db, headline.url);
    if (existing) continue;

    // Queue full article fetch
    await queueArticleFetch(sourceRecord.id, headline.url);
  }

  res.sendStatus(200);
});