compared with
This line was removed.
This word was removed. This word was added.
This line was added.

Changes (9)

View Page History


-- Combined mime type check and language detection on an arc file
register 'target/warcbase-0.1.0-SNAPSHOT-fatjar.jar';

-- Detect the mime type of the content using tika
a = foreach raw generate url,mime,content, org.warcbase.pig.piggybank.DetectMimeType(content) url,mime,content,SUBSTRING(date,0,12) as date,org.warcbase.pig.piggybank.DetectMimeType(content) as tikaMime;

-- Select the textual files
b = filter a by (tikaMime matches 'text.*'); == 'text/html');

-- Strip the tags from the content
c = foreach b generate url,mime,tikaMime,date,org.warcbase.pig.piggybank.ExtractRawText(content) as txt;

-- Use tika to identify the language of the textual content
d = foreach c generate url,mime,tikaMime,date,org.warcbase.pig.piggybank.DetectLanguage(txt) as lang,txt;

store d into 'tmp' using PigStorage();
-- Group by language and date
e = group d by (lang,date);

Example output from above script: [^output.txt]
-- Count the number of entries in the groups
f = foreach e generate $0, COUNT($1);

-- Order by the groups
-- WARNING! This is an expensive operation
g = ORDER f BY $1 DESC;

dump g;