scrape scrape scrape

totally half-finished thought. maybe it’ll spawn an idea for you… there’s a zillion+1 ways to scrape information from the web these days. here’s the easiest i’ve found:

require 'nokogiri'
require 'open-uri'
require 'tidy_ffi'

class CrappyScraper

	attr_accessor :doc	
	
	def search(keyword)
		@doc = Nokogiri::HTML(open("http://www.google.com/search?q=" + keyword))

		@doc.xpath('//h3/a').each do |node|
			puts node.text
		end

	end
	
	def scrape(url)
		@doc = Nokogiri::HTML(open(url))
		
		@doc.xpath('//span/a').each do |node|
  			puts node.text
		end
	end

	def write_clean(filename)
		File.open(filename, 'w') do |f| 
						doc_clean = TidyFFI::Tidy.new(@doc.to_s).clean
						f.write(doc_clean) 
		end
	end
	
	def to_s
		TidyFFI::Tidy.new(@doc.to_s).clean
	end
	
	def write(filename)
		File.open(filename, 'w') { |f| f.write(@doc) }
	end
end


x = CrappyScraper.new
x.search('cowabunga')
puts x.to_s