300 lines
11 KiB
SQL
300 lines
11 KiB
SQL
CREATE TABLE kvs (
|
|
id bigserial PRIMARY KEY,
|
|
t_update timestamp NOT NULL DEFAULT (now() at time zone 'utc'),
|
|
key varchar(200) NOT NULL UNIQUE,
|
|
value jsonb
|
|
)
|
|
----
|
|
COMMENT ON COLUMN kvs.t_update IS 'Time of last update or insert of the entry';
|
|
----
|
|
COMMENT ON COLUMN kvs.key IS 'Key';
|
|
----
|
|
COMMENT ON COLUMN kvs.value IS 'Value';
|
|
----
|
|
COMMENT ON TABLE kvs IS 'Simple key-value store';
|
|
----
|
|
INSERT INTO kvs (key, value) VALUES ('schema_version', '1');
|
|
----
|
|
CREATE TABLE site (
|
|
id bigserial PRIMARY KEY,
|
|
canonical_url varchar(200),
|
|
base_url varchar(200) NOT NULL,
|
|
base_urls varchar(200)[] NOT NULL,
|
|
domains varchar(100)[],
|
|
ips inet[] NULL,
|
|
crawl_enabled bool NOT NULL DEFAULT false,
|
|
crawl_active bool NOT NULL DEFAULT false,
|
|
next_full_crawl timestamp,
|
|
next_feed_crawl timestamp,
|
|
last_update timestamp,
|
|
last_pub timestamp,
|
|
pub_dates jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
langs char(2)[] NOT NULL DEFAULT ARRAY[]::varchar(2)[],
|
|
alt_langs jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
title varchar(200),
|
|
description varchar(2000),
|
|
keywords varchar(50)[] NOT NULL DEFAULT ARRAY[]::varchar(50)[],
|
|
linkbacks jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
meta_info jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
boilerplate_texts jsonb NOT NULL DEFAULT '[]'::jsonb
|
|
)
|
|
----
|
|
CREATE INDEX site__base_url ON site (base_url)
|
|
----
|
|
CREATE INDEX site__base_urls ON site (base_urls)
|
|
----
|
|
CREATE INDEX site__domains ON site (domains)
|
|
----
|
|
CREATE INDEX site__ips ON site (ips)
|
|
----
|
|
CREATE INDEX site__next_full_crawl ON site (next_full_crawl)
|
|
----
|
|
CREATE INDEX site__next_feed_crawl ON site (next_feed_crawl)
|
|
----
|
|
CREATE INDEX site__langs ON site (langs)
|
|
----
|
|
CREATE INDEX site__title ON site (title)
|
|
----
|
|
CREATE INDEX site__description ON site (description)
|
|
----
|
|
CREATE INDEX site__keywords ON site (keywords)
|
|
----
|
|
COMMENT ON COLUMN site.base_url IS 'Preferred base URLs (from column base_urls)'
|
|
----
|
|
COMMENT ON COLUMN site.base_urls IS 'Base URLs that have been found to return the same content'
|
|
----
|
|
COMMENT ON COLUMN site.domains IS 'Domains that have been found to return the same content'
|
|
----
|
|
COMMENT ON COLUMN site.ips IS 'IPv4 or IPv6 addresses of the hostnames in base_urls'
|
|
----
|
|
COMMENT ON COLUMN site.crawl_enabled IS 'Whether the site is should be indexed'
|
|
----
|
|
COMMENT ON COLUMN site.crawl_active IS 'Whether the crawl is in progress'
|
|
----
|
|
COMMENT ON COLUMN site.next_full_crawl IS 'Crawl all resources of this site again after this instant of time; do not crawl if null'
|
|
----
|
|
COMMENT ON COLUMN site.next_feed_crawl IS 'Crawl the feed resources of this site again after this instant of time; do not crawl if null'
|
|
----
|
|
COMMENT ON COLUMN site.last_update IS 'Time of last update of this site (in this database)'
|
|
----
|
|
COMMENT ON COLUMN site.last_pub IS 'Estimated time of last content publication on the site'
|
|
----
|
|
COMMENT ON COLUMN site.pub_dates IS 'Change history: map visit date to estimated publication date'
|
|
----
|
|
COMMENT ON COLUMN site.langs IS 'Languages of the site (ISO 639-1 codes)'
|
|
----
|
|
COMMENT ON COLUMN site.alt_langs IS 'Map links to alternative language versions of the site to ISO 639-1 languages codes'
|
|
----
|
|
COMMENT ON COLUMN site.title IS 'Title as obtained from title tag or meta tags'
|
|
----
|
|
COMMENT ON COLUMN site.description IS 'Description as obtained from meta tags'
|
|
----
|
|
COMMENT ON COLUMN site.keywords IS 'Keywords as obtained from meta tags'
|
|
----
|
|
COMMENT ON COLUMN site.linkbacks IS 'Map URL to type of linkback (cf. https://en.wikipedia.org/wiki/Linkback)'
|
|
----
|
|
COMMENT ON COLUMN site.meta_info IS 'Values from meta tags and other meta information'
|
|
----
|
|
COMMENT ON COLUMN site.boilerplate_texts IS 'Boilerplate texts on the startpage and other sample pages'
|
|
----
|
|
COMMENT ON TABLE site IS 'Website'
|
|
----
|
|
CREATE TABLE site_queue (
|
|
id bigserial PRIMARY KEY,
|
|
src bigint NULL REFERENCES site(id) ON DELETE CASCADE,
|
|
url varchar(200) NOT NULL,
|
|
link_text varchar(100),
|
|
t_create timestamp NOT NULL DEFAULT (now() at time zone 'utc')
|
|
)
|
|
----
|
|
CREATE INDEX site_queue__url ON site_queue (url)
|
|
----
|
|
COMMENT ON COLUMN site_queue.src IS 'The id of the linking site; null in case of seeds or manual additions'
|
|
----
|
|
COMMENT ON COLUMN site_queue.url IS 'Base URL of site to be assessed, ending with a slash or a mandatory base path'
|
|
----
|
|
COMMENT ON COLUMN site_queue.link_text IS 'Text under the anchor tag on the source site'
|
|
----
|
|
COMMENT ON COLUMN site_queue.t_create IS 'Creation time of this entry'
|
|
----
|
|
COMMENT ON TABLE site_queue IS 'Queued site URLs'
|
|
----
|
|
CREATE TABLE site_feed (
|
|
id bigserial PRIMARY KEY,
|
|
site_id bigint NOT NULL REFERENCES site(id) ON DELETE CASCADE,
|
|
url varchar(200) NOT NULL,
|
|
etag text,
|
|
modified varchar(50),
|
|
t_visit timestamp,
|
|
t_content timestamp,
|
|
version varchar(10),
|
|
title varchar(200),
|
|
description text,
|
|
fail_count smallint NOT NULL DEFAULT 0
|
|
)
|
|
----
|
|
CREATE INDEX site_feed__site ON site_feed (site_id)
|
|
----
|
|
CREATE INDEX site_feed__t_content ON site_feed (t_content)
|
|
----
|
|
COMMENT ON COLUMN site_feed.site_id IS 'Id of the site on which this feed was found'
|
|
----
|
|
COMMENT ON COLUMN site_feed.url IS 'URL of the feed'
|
|
----
|
|
COMMENT ON COLUMN site_feed.etag IS 'Etag obtained when requesting the feed'
|
|
----
|
|
COMMENT ON COLUMN site_feed.modified IS 'Last-Modified HTTP header value obtained when requesting the feed'
|
|
----
|
|
COMMENT ON COLUMN site_feed.t_visit IS 'Time of last retrieval of the feed; null before first retrival'
|
|
----
|
|
COMMENT ON COLUMN site_feed.t_content IS 'Time of last content update; null before first retrieval'
|
|
----
|
|
COMMENT ON COLUMN site_feed.version IS 'Version of the feed; null before first retrival'
|
|
----
|
|
COMMENT ON COLUMN site_feed.title IS 'Title of the feed; null before first retrival'
|
|
----
|
|
COMMENT ON COLUMN site_feed.description IS 'Description of the feed; null before first retrival'
|
|
----
|
|
COMMENT ON COLUMN site_feed.fail_count IS 'Number of failed retrievals after last successful retrieval; zero before first retrival'
|
|
----
|
|
CREATE TABLE site_link (
|
|
id bigserial PRIMARY KEY,
|
|
src bigint NOT NULL REFERENCES site(id) ON DELETE CASCADE,
|
|
dst bigint NOT NULL REFERENCES site(id) ON DELETE CASCADE,
|
|
t_create timestamp NOT NULL DEFAULT (now() at time zone 'utc'),
|
|
link_text varchar(100)
|
|
)
|
|
----
|
|
ALTER TABLE site_link ADD CONSTRAINT site_link_edge UNIQUE (src, dst)
|
|
----
|
|
CREATE INDEX site_link__src ON site_link (src)
|
|
----
|
|
CREATE INDEX site_link__dst ON site_link (dst)
|
|
----
|
|
COMMENT ON COLUMN site_link.src IS 'Source site'
|
|
----
|
|
COMMENT ON COLUMN site_link.dst IS 'Destination site'
|
|
----
|
|
COMMENT ON COLUMN site_link.t_create IS 'Time of creation of this entry'
|
|
----
|
|
COMMENT ON COLUMN site_link.link_text IS 'Text under the anchor tag on the source site'
|
|
----
|
|
COMMENT ON TABLE site_link IS 'Cross-site link'
|
|
----
|
|
CREATE TABLE resource (
|
|
id bigserial PRIMARY KEY,
|
|
simhash bigint,
|
|
content_type varchar(50),
|
|
last_change timestamp,
|
|
text_len int,
|
|
lang char(2),
|
|
title varchar(200),
|
|
summary varchar(2000)
|
|
)
|
|
----
|
|
COMMENT ON COLUMN resource.simhash IS 'Simhash of the text content of the resource'
|
|
----
|
|
COMMENT ON COLUMN resource.content_type IS 'Content type extracted from Content-Type HTTP header'
|
|
----
|
|
COMMENT ON COLUMN resource.last_change IS 'Estimated time of the last update of this resource'
|
|
----
|
|
COMMENT ON COLUMN resource.text_len IS 'Length of the extracted text in characters'
|
|
----
|
|
COMMENT ON COLUMN resource.lang IS 'Language ISO 639-1 code'
|
|
----
|
|
COMMENT ON COLUMN resource.title IS 'Title of the resource (used for feed resources)'
|
|
----
|
|
COMMENT ON COLUMN resource.summary IS 'Content summary of the resource (used for feed resources)'
|
|
----
|
|
COMMENT ON TABLE resource IS 'Text resource (may be reachable by more than one path of a site)'
|
|
----
|
|
CREATE TABLE site_path (
|
|
id bigserial PRIMARY KEY,
|
|
site_id bigint NOT NULL REFERENCES site(id) ON DELETE CASCADE,
|
|
path varchar(400) NOT NULL,
|
|
last_visit timestamp,
|
|
filtered bool NOT NULL DEFAULT false,
|
|
ok_count smallint NOT NULL DEFAULT 0,
|
|
canonical bool,
|
|
resource_id bigint REFERENCES resource(id) ON DELETE CASCADE
|
|
)
|
|
----
|
|
ALTER TABLE site_path ADD CONSTRAINT site_path__unique UNIQUE (site_id, path)
|
|
----
|
|
CREATE INDEX site_path__site_path ON site_path (site_id, path)
|
|
----
|
|
CREATE INDEX site_path__resource ON site_path (resource_id)
|
|
----
|
|
COMMENT ON COLUMN site_path.site_id IS 'Site id'
|
|
----
|
|
COMMENT ON COLUMN site_path.path IS 'Path'
|
|
----
|
|
COMMENT ON COLUMN site_path.last_visit IS 'Time of last retrieval of the resource; null before first retrival'
|
|
----
|
|
COMMENT ON COLUMN site_path.filtered IS 'Whether the path was filtered'
|
|
----
|
|
COMMENT ON COLUMN site_path.ok_count IS 'Increased by 1 for every successful retrieval of the resource and decreased by 1 for every failed'
|
|
----
|
|
COMMENT ON COLUMN site_path.canonical IS 'Whether the path is the canonical one for the resource; null before first retrival'
|
|
----
|
|
COMMENT ON COLUMN site_path.resource_id IS 'Resource id; null before first retrieval'
|
|
----
|
|
COMMENT ON TABLE site_path IS 'Paths of a site pointing to text resources'
|
|
----
|
|
CREATE TABLE crawl (
|
|
id bigserial PRIMARY KEY,
|
|
site_id bigint NOT NULL REFERENCES site(id) ON DELETE CASCADE,
|
|
is_full bool NOT NULL DEFAULT false,
|
|
t_begin timestamp,
|
|
t_end timestamp,
|
|
n_resources int NOT NULL DEFAULT 0,
|
|
n_resources_new int NOT NULL DEFAULT 0
|
|
)
|
|
----
|
|
CREATE INDEX crawl__site ON crawl (site_id)
|
|
----
|
|
CREATE INDEX crawl__t_begin ON crawl (t_begin)
|
|
----
|
|
COMMENT ON COLUMN crawl.site_id IS 'Site that is being crawled'
|
|
----
|
|
COMMENT ON COLUMN crawl.is_full IS 'Whether the crawl is a full crawl; if not it is a feed crawl'
|
|
----
|
|
COMMENT ON COLUMN crawl.t_begin IS 'Begin time of the crawl'
|
|
----
|
|
COMMENT ON COLUMN crawl.t_end IS 'End time of the crawl; if t_end is null resuming a crawl will fetch all resources with last_visit before t_begin'
|
|
----
|
|
COMMENT ON COLUMN crawl.n_resources IS 'Number of resources that were fetched during the crawl'
|
|
----
|
|
COMMENT ON COLUMN crawl.n_resources_new IS 'Number of new resources found during the crawl'
|
|
----
|
|
COMMENT ON TABLE resource IS 'Crawl of resources on a site'
|
|
----
|
|
CREATE TYPE site_annotation_type AS ENUM ('whitelist', 'blacklist', 'suggestion', 'review', 'audience', 'location', 'themes', 'timescale')
|
|
----
|
|
COMMENT ON TYPE site_annotation_type IS 'Type of site annotation'
|
|
----
|
|
CREATE TABLE site_annotation (
|
|
id bigserial PRIMARY KEY,
|
|
site_id bigint REFERENCES site(id) ON DELETE SET NULL,
|
|
base_url varchar(200) NOT NULL,
|
|
ann_type site_annotation_type NOT NULL,
|
|
ann_content JSONB,
|
|
t_update timestamp NOT NULL DEFAULT (now() at time zone 'utc')
|
|
)
|
|
----
|
|
CREATE INDEX site_annotation__site ON site_annotation (site_id)
|
|
----
|
|
CREATE INDEX site_annotation__base_url ON site_annotation (base_url)
|
|
----
|
|
COMMENT ON COLUMN site_annotation.site_id IS 'Site that is being annotated'
|
|
----
|
|
COMMENT ON COLUMN site_annotation.base_url IS 'Base URL of the site being annotated'
|
|
----
|
|
COMMENT ON COLUMN site_annotation.ann_type IS 'Annotation type'
|
|
----
|
|
COMMENT ON COLUMN site_annotation.ann_content IS 'Annotation content'
|
|
----
|
|
COMMENT ON COLUMN site_annotation.t_update IS 'Time of last update'
|
|
----
|
|
COMMENT ON TABLE site_annotation IS 'Manual annotations on a site'
|