v / examples / web_crawler
Raw file | 25 loc (24 sloc) | 1000 bytes | Latest commit hash 017ace6ea
1import net.http
2import net.html
3
4fn main() {
5 // http.fetch() sends an HTTP request to the URL with the given method and configurations.
6 config := http.FetchConfig{
7 user_agent: 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
8 }
9 resp := http.fetch(http.FetchConfig{ ...config, url: 'https://tuicool.com' }) or {
10 println('failed to fetch data from the server')
11 return
12 }
13 // html.parse() parses and returns the DOM from the given text.
14 mut doc := html.parse(resp.body)
15 // html.DocumentObjectModel.get_tags_by_attribute_value() retrieves all tags in the document that have the given attribute name and value.
16 tags := doc.get_tags_by_attribute_value('class', 'list_article_item')
17 for tag in tags {
18 el := tag.children[1].children[0].children[0].children[0]
19 href := el.attributes['href'] or { panic('key not found') }
20 title := el.attributes['title'] or { panic('key not found') }
21 println('href: ${href}')
22 println('title: ${title}')
23 println('')
24 }
25}