Я пытаюсь использовать spaCy для создания новой категории объекта «MYPRODUCT» со списком названий продуктов.После обновления модели и сохранения на диск старые теги забываются.Это правильный способ сделать это.Я использовал некоторые из существующих примеров и создал свой скрипт.Кто-нибудь может мне помочь?
Мой код:
# training data
TRAIN_DATA = [
('Smartstream is a Infor Product.', {'entities': [(0, 11, 'MYPRODUCT')]}),
("Horses are too tall and they pretend to care about your feelings", {'entities': [(0, 6, 'ANIMAL')]}),
("Do they bite?", {'entities': [] }),
("horses are too tall and they pretend to care about your feelings", { 'entities': [(0, 6, 'ANIMAL')] }),
("horses pretend to care about your feelings", { 'entities': [(0, 6, 'ANIMAL')] }),
("horses?", {'entities': [(0, 6, 'ANIMAL')]}),
("General",{"entities":[(0,7, 'MYPRODUCT')]}),
("SmartStream",{"entities":[(0,11, 'MYPRODUCT')]}),
("TotalHR",{"entities":[(0,7, 'MYPRODUCT')]}),
("E Series",{"entities":[(0,8, 'MYPRODUCT')]}),
("M Series",{"entities":[(0,8, 'MYPRODUCT')]}),
("Infor Support Portal - Customer Care",{"entities":[(0,36, 'MYPRODUCT')]}),
("ALLTAX",{"entities":[(0,6, 'MYPRODUCT')]}),
("Infor Partner Network",{"entities":[(0,21, 'MYPRODUCT')]}),
("Host General",{"entities":[(0,12, 'MYPRODUCT')]}),
("StreamLine",{"entities":[(0,10, 'MYPRODUCT')]}),
("System 21",{"entities":[(0,9, 'MYPRODUCT')]}),
("TIMS",{"entities":[(0,4, 'MYPRODUCT')]}),
("i2",{"entities":[(0,2, 'MYPRODUCT')]}),
("EnRoute",{"entities":[(0,7, 'MYPRODUCT')]}),
("Help - zSeries",{"entities":[(0,14, 'MYPRODUCT')]}),
("Expense Management",{"entities":[(0,18, 'MYPRODUCT')]}),
("PM",{"entities":[(0,2, 'MYPRODUCT')]}),
("Pathway",{"entities":[(0,7, 'MYPRODUCT')]}),
("Education",{"entities":[(0,9, 'MYPRODUCT')]}),
("Anael",{"entities":[(0,5, 'MYPRODUCT')]}),
("Library Solutions",{"entities":[(0,17, 'MYPRODUCT')]}),
("HR Tax & Reg",{"entities":[(0,12, 'MYPRODUCT')]}),
("Pegasus",{"entities":[(0,7, 'MYPRODUCT')]}),
("F9",{"entities":[(0,2, 'MYPRODUCT')]}),
("SunSystems",{"entities":[(0,10, 'MYPRODUCT')]}),
("Q&A/Vision",{"entities":[(0,10, 'MYPRODUCT')]}),
("Elevon",{"entities":[(0,6, 'MYPRODUCT')]}),
("MAX",{"entities":[(0,3, 'MYPRODUCT')]}),
("Prism",{"entities":[(0,5, 'MYPRODUCT')]}),
("Protean",{"entities":[(0,7, 'MYPRODUCT')]}),
("SXe",{"entities":[(0,3, 'MYPRODUCT')]}),
("Unison",{"entities":[(0,6, 'MYPRODUCT')]}),
("PRMS",{"entities":[(0,4, 'MYPRODUCT')]}),
("IRONSIDE",{"entities":[(0,8, 'MYPRODUCT')]}),
("Epiphany Sales and Service",{"entities":[(0,26, 'MYPRODUCT')]}),
("BI/Cognos",{"entities":[(0,9, 'MYPRODUCT')]}),
("FMS Masterpiece",{"entities":[(0,15, 'MYPRODUCT')]}),
("ERP LX",{"entities":[(0,6, 'MYPRODUCT')]}),
("BOSS",{"entities":[(0,4, 'MYPRODUCT')]}),
("INFINIUM",{"entities":[(0,8, 'MYPRODUCT')]}),
("SCM WM",{"entities":[(0,6, 'MYPRODUCT')]}),
("SCEM",{"entities":[(0,4, 'MYPRODUCT')]}),
("Epiphany Interaction Advisor",{"entities":[(0,28, 'MYPRODUCT')]}),
("Epiphany Marketing",{"entities":[(0,18, 'MYPRODUCT')]}),
("KBM",{"entities":[(0,3, 'MYPRODUCT')]}),
("TMS",{"entities":[(0,3, 'MYPRODUCT')]}),
("iSeries EAM",{"entities":[(0,11, 'MYPRODUCT')]}),
("Demand Planning",{"entities":[(0,15, 'MYPRODUCT')]}),
("CAS",{"entities":[(0,3, 'MYPRODUCT')]}),
("PROVIA",{"entities":[(0,6, 'MYPRODUCT')]})
]
@plac.annotations(
model=("Model name. Defaults to blank 'en' model.", "option", "m", str),
output_dir=("Optional output directory", "option", "o", Path),
n_iter=("Number of training iterations", "option", "n", int))
def main(model="en", output_dir="c:\\myproject\\models", n_iter=10):
"""Load the model, set up the pipeline and train the entity recognizer."""
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank('en') # create blank Language class
print("Created blank 'en' model")
# create the built-in pipeline components and add them to the pipeline
# nlp.create_pipe works for built-ins that are registered with spaCy
if 'ner' not in nlp.pipe_names:
ner = nlp.create_pipe('ner')
nlp.add_pipe(ner, last=True)
# otherwise, get it so we can add labels
else:
ner = nlp.get_pipe('ner')
# add labels
ner.add_label('MYPRODUCT')
ner.add_label('ANIMAL') # add new entity label to entity recognizer
ner.add_label('GOODBYE')
ner.add_label('CURSE')
ner.add_label('AFFIRM')
ner.add_label('GREETINGS')
ner.add_label('LOC-Q')
ner.add_label('WHQ')
ner.add_label('PERSON')
# get names of other pipes to disable them during training
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
with nlp.disable_pipes(*other_pipes): # only train NER
optimizer = nlp.begin_training()
for itn in range(n_iter):
random.shuffle(TRAIN_DATA)
losses = {}
batches = minibatch(TRAIN_DATA, size=compounding(4., 32., 1.001))
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(
texts, # batch of texts
annotations, # batch of annotations
drop=0.5, # dropout - make it harder to memorise data
sgd=optimizer, # callable to update weights
losses=losses)
print('Losses', losses)
# test the trained model
for text, _ in TRAIN_DATA:
doc = nlp(text)
print('Entities', [(ent.text, ent.label_) for ent in doc.ents])
print('Tokens', [(t.text, t.ent_type_, t.ent_iob) for t in doc])
# save model to output directory
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the saved model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
for text, _ in TRAIN_DATA:
doc = nlp2(text)
print('Entities-Test', [(ent.text, ent.label_) for ent in doc.ents])
print('Tokens-Test', [(t.text, t.ent_type_, t.ent_iob) for t in doc])
if __name__ == '__main__':
plac.call(main)
Есть ли способ сделать это с помощью Matcher или PatternMatcher?