eu.dnetlib.data.mapreduce.hbase.oai

Class OaiFeedMapper

java.lang.Object

org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>

eu.dnetlib.data.mapreduce.hbase.oai.OaiFeedMapper

public class OaiFeedMapper
extends org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.hadoop.mapreduce.Mapper

org.apache.hadoop.mapreduce.Mapper.Context

Constructor Summary

Constructors

Constructor and Description
OaiFeedMapper()

Method Summary

Modifier and Type	Method and Description
boolean	checkRecordFields(com.google.common.collect.Multimap<String,String> recordFields, org.apache.hadoop.mapreduce.Mapper.Context context, String recordKey, String recordBody)
protected void	cleanup(org.apache.hadoop.mapreduce.Mapper.Context context)
protected com.mongodb.DBObject	createBasicObject(String oaiID, String record, com.google.common.collect.Multimap<String,String> recordProperties, org.apache.hadoop.mapreduce.Mapper.Context context)
org.bson.types.Binary	createCompressRecord(org.apache.hadoop.mapreduce.Mapper.Context context, String recordKey, String recordBody)
com.mongodb.client.MongoCollection<com.mongodb.DBObject>	getCollection()
com.mongodb.client.MongoCollection<com.mongodb.DBObject>	getDiscardedCollection()
String	getDuplicateXPath()
Date	getFeedDate()
MongoSetCollection	getMongoSetCollection()
OAIConfiguration	getOaiConfiguration()
OAIConfigurationStringReader	getOaiConfigurationReader()
String[]	getParseDatePatterns()
boolean	isSkipDuplicates()
protected void	map(org.apache.hadoop.io.Text key, org.apache.hadoop.io.Text value, org.apache.hadoop.mapreduce.Mapper.Context context)
protected Date	parseDate(String date)
void	setCollection(com.mongodb.client.MongoCollection<com.mongodb.DBObject> collection)
void	setDiscardedCollection(com.mongodb.client.MongoCollection<com.mongodb.DBObject> discardedCollection)
void	setDuplicateXPath(String duplicateXPath)
void	setFeedDate(Date feedDate)
void	setMongoSetCollection(MongoSetCollection mongoSetCollection)
void	setOaiConfiguration(OAIConfiguration oaiConfiguration)
void	setOaiConfigurationReader(OAIConfigurationStringReader oaiConfigurationReader)
void	setParseDatePatterns(String[] parseDatePatterns)
void	setSkipDuplicates(boolean skipDuplicates)
protected void	setup(org.apache.hadoop.mapreduce.Mapper.Context context)

Methods inherited from class org.apache.hadoop.mapreduce.Mapper

run

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

OaiFeedMapper

public OaiFeedMapper()

Method Detail

setup

protected void setup(org.apache.hadoop.mapreduce.Mapper.Context context)
              throws UnknownHostException

Overrides:

setup in class org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>

Throws:

UnknownHostException

map

protected void map(org.apache.hadoop.io.Text key,
                   org.apache.hadoop.io.Text value,
                   org.apache.hadoop.mapreduce.Mapper.Context context)
            throws IOException,
                   InterruptedException

Overrides:

map in class org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>

Throws:

IOException

InterruptedException

checkRecordFields

public boolean checkRecordFields(com.google.common.collect.Multimap<String,String> recordFields,
                                 org.apache.hadoop.mapreduce.Mapper.Context context,
                                 String recordKey,
                                 String recordBody)

parseDate

protected Date parseDate(String date)

createBasicObject

protected com.mongodb.DBObject createBasicObject(String oaiID,
                                                 String record,
                                                 com.google.common.collect.Multimap<String,String> recordProperties,
                                                 org.apache.hadoop.mapreduce.Mapper.Context context)

createCompressRecord

public org.bson.types.Binary createCompressRecord(org.apache.hadoop.mapreduce.Mapper.Context context,
                                                  String recordKey,
                                                  String recordBody)

cleanup

protected void cleanup(org.apache.hadoop.mapreduce.Mapper.Context context)
                throws IOException,
                       InterruptedException

Overrides:

cleanup in class org.apache.hadoop.mapreduce.Mapper<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text,org.apache.hadoop.io.NullWritable,org.apache.hadoop.io.NullWritable>

Throws:

IOException

InterruptedException

getCollection

public com.mongodb.client.MongoCollection<com.mongodb.DBObject> getCollection()

setCollection

public void setCollection(com.mongodb.client.MongoCollection<com.mongodb.DBObject> collection)

getDiscardedCollection

public com.mongodb.client.MongoCollection<com.mongodb.DBObject> getDiscardedCollection()

setDiscardedCollection

public void setDiscardedCollection(com.mongodb.client.MongoCollection<com.mongodb.DBObject> discardedCollection)

getOaiConfigurationReader

public OAIConfigurationStringReader getOaiConfigurationReader()

setOaiConfigurationReader

public void setOaiConfigurationReader(OAIConfigurationStringReader oaiConfigurationReader)

getOaiConfiguration

public OAIConfiguration getOaiConfiguration()

setOaiConfiguration

public void setOaiConfiguration(OAIConfiguration oaiConfiguration)

getFeedDate

public Date getFeedDate()

setFeedDate

public void setFeedDate(Date feedDate)

getMongoSetCollection

public MongoSetCollection getMongoSetCollection()

setMongoSetCollection

public void setMongoSetCollection(MongoSetCollection mongoSetCollection)

getDuplicateXPath

public String getDuplicateXPath()

setDuplicateXPath

public void setDuplicateXPath(String duplicateXPath)

isSkipDuplicates

public boolean isSkipDuplicates()

setSkipDuplicates

public void setSkipDuplicates(boolean skipDuplicates)

getParseDatePatterns

public String[] getParseDatePatterns()

setParseDatePatterns

public void setParseDatePatterns(String[] parseDatePatterns)