Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- df.info()
- <class 'dask.dataframe.core.DataFrame'>
- Columns: 5 entries, claim_no to litigation
- dtypes: object(2), int64(3)
- claim_no claim_txt I CL ICC lit
- 0 8697278-17 battery comprising interior battery active ele... 106 2 0
- >>tagged_document[0]
- >>TaggedDocument(words=['battery', 'comprising', 'interior', 'battery', 'active', 'elements', 'battery', 'cell', 'casing', 'said', 'cell', 'casing', 'comprising', 'first', 'casing', 'element', 'first', 'contact', 'surface', 'second', 'casing', 'element', 'second', 'contact', 'surface', 'wherein', 'assembled', 'position', 'first', 'second', 'contact', 'surfaces', 'contact', 'first', 'second', 'casing', 'elements', 'encase', 'active', 'materials', 'battery', 'cell', 'interior', 'space', 'wherein', 'least', 'one', 'gas', 'tight', 'seal', 'layer', 'arranged', 'first', 'second', 'contact', 'surfaces', 'seal', 'interior', 'space', 'characterized', 'one', 'first', 'second', 'contact', 'surfaces', 'comprises', 'electrically', 'insulating', 'void', 'volume', 'layer', 'first', 'second', 'contact', 'surfaces', 'comprises', 'formable', 'material', 'layer', 'fills', 'voids', 'surface', 'void', 'volume', 'layer', 'hermetically', 'assembled', 'position', 'form', 'seal', 'layer'], tags=['8697278-17'])
- >>len(tagged_document) == len(df['claim_txt'])
- def read_corpus_tag_sub(df,corp='claim_txt',tags=['claim_no']):
- for i, line in enumerate(df[corp]):
- yield gensim.models.doc2vec.TaggedDocument(gensim.utils.simple_preprocess(line), (list(df.loc[i,tags].values)))
- tagged_document = df.map_partitions(read_corpus_tag_sub,meta=TaggedDocument)
- tagged_document = tagged_document.compute()
- def read_corpus_tag_sub(df,corp='claim_txt',tags=['claim_no']):
- for i, line in enumerate(df[corp]):
- return gensim.models.doc2vec.TaggedDocument(gensim.utils.simple_preprocess(line), (list(df.loc[i,tags].values)))
- tagged_document = df.map_partitions(read_corpus_tag_sub,meta=TaggedDocument)
- tagged_document = tagged_document.compute()
- def read_corpus_tag_sub(df,corp='claim_txt',tags=['claim_no']):
- tagged_list = []
- for i, line in enumerate(df[corp]):
- tagged = gensim.models.doc2vec.TaggedDocument(gensim.utils.simple_preprocess(line), (list(df.loc[i,tags].values)))
- tagged_list.append(tagged)
- return tagged_list
- def tag_corp(corp,tag):
- return gensim.models.doc2vec.TaggedDocument(gensim.utils.simple_preprocess(corp), ([tag]))
- tagged_document = [tag_corp(x,y) for x,y in list(zip(df_smple['claim_txt'],df_smple['claim_no']))]
- tagged_document = list(read_corpus_tag_sub(df))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement