en/AdvancedNlpCourse2020/LanguageResourcesFromWeb: plagiarism_simple.py

#!/usr/bin/python

r"""
IA161, autumn 2019, homework on Building Language Resources from the Web.
This is a basic script for simple plagiarism detection that implements
relative frequency of words. You can add e.g. the following to improve it:
- change function doc_similarity,
- make it work with sentences rather than whole documents,
- add another method of duplicate detection,
- use multiple methods to decide if a sample is a duplicate.
Input: vertical (one token per line),
- structure <doc/> with attributes author, id, class, source,
- 3 columns: word, tag, lemma+POS.
Output:
- for each plagiarism document: id, id of detected source, id of the actual source,
- evaluation of precision, recall, F1-measure.
Usage (processing pipeline on asteria04):
ssh aurora.fi.muni.cz
ssh asteria04
cat *.txt | /opt/majka_pipe/majka-czech_v2.sh | cut -f1-3 | python plagiarism_simple.py
cat *.txt | /opt/treetagger_pipe/tt-english_v2.1.sh | python plagiarism_simple.py
"""

import sys, codecs, re
stdin = codecs.getreader('utf-8')(sys.stdin)
stdout = codecs.getwriter('utf-8')(sys.stdout)
header_re = re.compile('<doc author="([^"]+)" id="(\d+)" class="(plagiarism|original)" source="(\d+)"')

#Nacteme vsechny dokumenty do pameti (v malem poctu nevadi).
#Pro kazdy dokument vytvorime seznam slov s cetnosti.
doc_sets = {} #sady dokumentu, kazda sada od jednoho autora
doc = {}
for line in stdin:
    if line.startswith('<doc'):
        #Vytvorime strukturu pro uchovani obsahu a metadat dokumentu.
        author, id_, class_, source_id = header_re.match(line).groups()
        doc = {
            'author': author,
            'id': id_,
            'class': class_,
            'source_id': source_id,
            'wordlist': {},
        }
    elif line.startswith('</doc'):
        #Pridame dokument do sady daneho autora mezi originalni nebo podezrele dokumenty.
        if not doc['author'] in doc_sets:
            doc_sets[doc['author']] = {'original': [], 'suspicious': []}
        if doc['class'] == 'original':
            doc_sets[doc['author']]['original'].append(doc)
        else:
            doc_sets[doc['author']]['suspicious'].append(doc)
    elif not line.startswith('<'):
        #Pridame vyskyt slova do seznamu slov s poctem vyskytu v dokumentu.
        word, tag, lemma = line.rstrip().split('\t')[:3]
        doc['wordlist'][word] = doc['wordlist'].get(word, 0) + 1

#Tuto funkci muzete vylepsit nebo implementovat jinak,
#v teto podobe prevede dokumenty na vektory a spocita jejich vzadelnost,
#kazda slozka vektoru reprezentuje jedno slovo, hodnota je relativni cetnost slova,
#pri tomto srovani dokumentu tedy zalezi jen na relativni cetnosti slov.
#Vstup: dva dokumenty.
#Vystup: cislo mezi 0 a 1 udavajici, jak jsou dokumenty podobne (1 = identicke).
DOC_SIMILARITY_THRESHOLD = 0.5
from scipy import spatial
def doc_similarity(doc1, doc2):
    vector1, vector2 = [], []
    all_words = list(doc1['wordlist'].keys() + doc2['wordlist'].keys())
    doc1_len = float(sum(doc1['wordlist'].values()))
    doc2_len = float(sum(doc2['wordlist'].values()))
    for word in all_words:
        vector1.append(doc1['wordlist'].get(word, 0) / doc1_len)
        vector2.append(doc2['wordlist'].get(word, 0) / doc2_len)
    cosine_similarity = 1.0 - spatial.distance.cosine(vector1, vector2)
    return cosine_similarity

#Srovname wordlisty podezrelych dokumentu s originaly ze stejne sady dokumentu.
#Zaroven vyhodnocujeme uspesnost.
stats = {'tp': 0, 'fp': 0, 'tn': 0, 'fn': 0}
for author, doc_set in doc_sets.iteritems():
    stdout.write(u'Doc set by %s\n' % author)
    set_stats = {'tp': 0, 'fp': 0, 'tn': 0, 'fn': 0}
    for doc in doc_set['suspicious']:
        #srovnani se vsemi originaly
        most_similar_doc_id = doc['id'] #vychozi stav je dokument je nejpodobnejsi sam sobe
        highest_similarity_score = 0.0
        plagiarism = False
        for orig_doc in doc_set['original']:
            similarity_score = doc_similarity(doc, orig_doc)
            if similarity_score >= DOC_SIMILARITY_THRESHOLD \
                    and similarity_score > highest_similarity_score:
                most_similar_doc_id = orig_doc['id']
                highest_similarity_score = similarity_score
                plagiarism = True
        stdout.write(u'%s\t%s\t%s\n' % (doc['id'], most_similar_doc_id, doc['source_id']))
        #vyhodnoceni
        if most_similar_doc_id == doc['source_id']:
            if doc['class'] == 'plagiarism':
                set_stats['tp'] += 1
            else:
                set_stats['tn'] += 1
        else:
            if doc['class'] == 'plagiarism':
                set_stats['fp'] += 1
            else:
                set_stats['fn'] += 1
    #vyhodnoceni
    try:
        precision = set_stats['tp'] / float(set_stats['tp'] + set_stats['fp'])
    except ZeroDivisionError:
        precision = 0.0
    try:
        recall = set_stats['tp'] / float(set_stats['tp'] + set_stats['fn'])
    except ZeroDivisionError:
        recall = 0.0
    try:
        f1_measure = 2 * precision * recall / (precision + recall)
    except ZeroDivisionError:
        f1_measure = 0.0
    stdout.write(u'Set precision: %.2f, recall: %.2f, F1: %.2f\n\n' %
        (precision, recall, f1_measure))
    stats['tp'] += set_stats['tp']
    stats['fp'] += set_stats['fp']
    stats['tn'] += set_stats['tn']
    stats['fn'] += set_stats['fn']
try:
    precision = stats['tp'] / float(stats['tp'] + stats['fp'])
except ZeroDivisionError:
    precision = 0.0
try:
    recall = stats['tp'] / float(stats['tp'] + stats['fn'])
except ZeroDivisionError:
    recall = 0.0
try:
    f1_measure = 2 * precision * recall / (precision + recall)
except ZeroDivisionError:
    f1_measure = 0.0
stdout.write(u'Overall precision: %.2f, recall: %.2f, F1: %.2f\n' %
    (precision, recall, f1_measure))