Mongo 3.6: удаление дубликатов - PullRequest
Новая
       22

Mongo 3.6: удаление дубликатов

0 голосов
/ 03 июля 2018

Используя следующую команду, я получаю все повторяющиеся значения в моей коллекции на основе поля «нет»: 

db.books.aggregate([{ $group: { _id: {no: "$no"}, uniqueIds: {$addToSet: "$_id"}, count: {$sum: 1}}}, { $match: { count: {"$gt": 1}}}, { $sort: {count: -1}}])

Возвращает следующее: 

{ "_id" : { "no" : 160541455 }, "uniqueIds" : [ ObjectId("5b3a3a3bd6194c5bfe40a523"), ObjectId("5b3a345dd6194c5bfe3deefa"), ObjectId("5b3a3509d6194c5bfe3e2e29"), ObjectId("5b3a3440d6194c5bfe3de3b4"), ObjectId("5b3a336ad6194c5bfe3d80a9"), ObjectId("5b3a3600d6194c5bfe3eb073"), ObjectId("5b3a37e1d6194c5bfe3f96f0"), ObjectId("5b3a35aed6194c5bfe3e8b83"), ObjectId("5b3a36ead6194c5bfe3f2dbd"), ObjectId("5b3a3ea6d6194c5bfe42d794"), ObjectId("5b3a3731d6194c5bfe3f50d2"), ObjectId("5b3a399bd6194c5bfe40676c") ], "count" : 12 }
{ "_id" : { "no" : 105900593 }, "uniqueIds" : [ ObjectId("5b3a3efad6194c5bfe42fc1f"), ObjectId("5b3a3df3d6194c5bfe429191"), ObjectId("5b3a3697d6194c5bfe3f0ca9"), ObjectId("5b3a385fd6194c5bfe3fdced"), ObjectId("5b3a3b69d6194c5bfe416901"), ObjectId("5b3a347ed6194c5bfe3dfeff"), ObjectId("5b3a335ad6194c5bfe3d7c15"), ObjectId("5b3a3887d6194c5bfe3feaad"), ObjectId("5b3a38b8d6194c5bfe400088"), ObjectId("5b3a358dd6194c5bfe3e81ea"), ObjectId("5b3a3b3fd6194c5bfe4157bc"), ObjectId("5b3a3decd6194c5bfe428efa") ], "count" : 12 }
{ "_id" : { "no" : 144919593 }, "uniqueIds" : [ ObjectId("5b3a3e46d6194c5bfe42b277"), ObjectId("5b3a3c22d6194c5bfe41b05a"), ObjectId("5b3a398cd6194c5bfe4065be"), ObjectId("5b3a37a1d6194c5bfe3f8239"), ObjectId("5b3a3678d6194c5bfe3f0496"), ObjectId("5b3a33d2d6194c5bfe3db42c"), ObjectId("5b3a36e6d6194c5bfe3f2d4d"), ObjectId("5b3a3d05d6194c5bfe421e97"), ObjectId("5b3a384ad6194c5bfe3fd621"), ObjectId("5b3a3565d6194c5bfe3e6c44") ], "count" : 10 }
{ "_id" : { "no" : 86204442 }, "uniqueIds" : [ ObjectId("5b3a3df5d6194c5bfe429204"), ObjectId("5b3a3d1cd6194c5bfe4224a1"), ObjectId("5b3a3b33d6194c5bfe415288"), ObjectId("5b3a3934d6194c5bfe403ed9"), ObjectId("5b3a3c39d6194c5bfe41b664"), ObjectId("5b3a37dbd6194c5bfe3f961c"), ObjectId("5b3a3688d6194c5bfe3f0613"), ObjectId("5b3a3731d6194c5bfe3f50c6"), ObjectId("5b3a37dfd6194c5bfe3f9697") ], "count" : 9 }
{ "_id" : { "no" : 119417424 }, "uniqueIds" : [ ObjectId("5b3a3b34d6194c5bfe4152a5"), ObjectId("5b3a392cd6194c5bfe403b21"), ObjectId("5b3a399ad6194c5bfe406736"), ObjectId("5b3a38b9d6194c5bfe4000c7"), ObjectId("5b3a3ea5d6194c5bfe42d77f"), ObjectId("5b3a374cd6194c5bfe3f5c9f"), ObjectId("5b3a33d9d6194c5bfe3db4e3"), ObjectId("5b3a3386d6194c5bfe3d8ab8"), ObjectId("5b3a3857d6194c5bfe3fd754") ], "count" : 9 }
{ "_id" : { "no" : 159374794 }, "uniqueIds" : [ ObjectId("5b3a3e52d6194c5bfe42b3b8"), ObjectId("5b3a3c0fd6194c5bfe41ac10"), ObjectId("5b3a38f5d6194c5bfe401b2f"), ObjectId("5b3a37afd6194c5bfe3f83b7"), ObjectId("5b3a33ebd6194c5bfe3dbc27"), ObjectId("5b3a36edd6194c5bfe3f2e78"), ObjectId("5b3a34ced6194c5bfe3e1b24"), ObjectId("5b3a3b36d6194c5bfe41530d"), ObjectId("5b3a3753d6194c5bfe3f5f99") ], "count" : 9 }
{ "_id" : { "no" : 159363089 }, "uniqueIds" : [ ObjectId("5b3a3e4fd6194c5bfe42b340"), ObjectId("5b3a3e0ad6194c5bfe429a1f"), ObjectId("5b3a3854d6194c5bfe3fd708"), ObjectId("5b3a34cbd6194c5bfe3e1acb"), ObjectId("5b3a38f4d6194c5bfe401afb"), ObjectId("5b3a36c4d6194c5bfe3f20aa"), ObjectId("5b3a3a06d6194c5bfe409119"), ObjectId("5b3a36ecd6194c5bfe3f2e2b"), ObjectId("5b3a3764d6194c5bfe3f6439") ], "count" : 9 }
{ "_id" : { "no" : 101412948 }, "uniqueIds" : [ ObjectId("5b3a3ea0d6194c5bfe42d708"), ObjectId("5b3a37b7d6194c5bfe3f8749"), ObjectId("5b3a3712d6194c5bfe3f4510"), ObjectId("5b3a334ed6194c5bfe3d759b"), ObjectId("5b3a3929d6194c5bfe403aca"), ObjectId("5b3a36f3d6194c5bfe3f334d"), ObjectId("5b3a39bad6194c5bfe407541"), ObjectId("5b3a3477d6194c5bfe3df66b"), ObjectId("5b3a35bdd6194c5bfe3e9233") ], "count" : 9 }
{ "_id" : { "no" : 107412155 }, "uniqueIds" : [ ObjectId("5b3a3e52d6194c5bfe42b3d1"), ObjectId("5b3a3a3bd6194c5bfe40a53a"), ObjectId("5b3a399cd6194c5bfe40678d"), ObjectId("5b3a390fd6194c5bfe402698"), ObjectId("5b3a3c0fd6194c5bfe41ac28"), ObjectId("5b3a346ad6194c5bfe3df338"), ObjectId("5b3a3584d6194c5bfe3e718a"), ObjectId("5b3a373cd6194c5bfe3f5563"), ObjectId("5b3a37e5d6194c5bfe3f97d6") ], "count" : 9 }
{ "_id" : { "no" : 156426829 }, "uniqueIds" : [ ObjectId("5b3a3cc6d6194c5bfe421829"), ObjectId("5b3a3f01d6194c5bfe430069"), ObjectId("5b3a3a29d6194c5bfe40a378"), ObjectId("5b3a37c9d6194c5bfe3f8fdc"), ObjectId("5b3a3558d6194c5bfe3e6b19"), ObjectId("5b3a3dabd6194c5bfe428666"), ObjectId("5b3a366ad6194c5bfe3f0356"), ObjectId("5b3a36e3d6194c5bfe3f2d02"), ObjectId("5b3a3798d6194c5bfe3f8160") ], "count" : 9 }
{ "_id" : { "no" : 104966976 }, "uniqueIds" : [ ObjectId("5b3a3c35d6194c5bfe41b614"), ObjectId("5b3a3a04d6194c5bfe4090e9"), ObjectId("5b3a3d19d6194c5bfe422451"), ObjectId("5b3a37a7d6194c5bfe3f82bc"), ObjectId("5b3a348cd6194c5bfe3e034c"), ObjectId("5b3a3e58d6194c5bfe42b5d2"), ObjectId("5b3a345cd6194c5bfe3deee2"), ObjectId("5b3a35aad6194c5bfe3e8b15"), ObjectId("5b3a36c3d6194c5bfe3f208b") ], "count" : 9 }
{ "_id" : { "no" : 105460396 }, "uniqueIds" : [ ObjectId("5b3a3e04d6194c5bfe429777"), ObjectId("5b3a3b99d6194c5bfe417eb6"), ObjectId("5b3a38c7d6194c5bfe400533"), ObjectId("5b3a3e6ed6194c5bfe42bbc6"), ObjectId("5b3a38f5d6194c5bfe401b1d"), ObjectId("5b3a3ee8d6194c5bfe42f43f"), ObjectId("5b3a39a5d6194c5bfe4069ec"), ObjectId("5b3a3a07d6194c5bfe409134") ], "count" : 8 }
{ "_id" : { "no" : 158805980 }, "uniqueIds" : [ ObjectId("5b3a3efad6194c5bfe42fc7c"), ObjectId("5b3a3abed6194c5bfe41470a"), ObjectId("5b3a337cd6194c5bfe3d89c3"), ObjectId("5b3a33bdd6194c5bfe3db218"), ObjectId("5b3a3dfcd6194c5bfe42955f"), ObjectId("5b3a3be7d6194c5bfe41a820"), ObjectId("5b3a3444d6194c5bfe3de564"), ObjectId("5b3a391fd6194c5bfe4039d5") ], "count" : 8 }
{ "_id" : { "no" : 107411546 }, "uniqueIds" : [ ObjectId("5b3a3c0fd6194c5bfe41ac2a"), ObjectId("5b3a3b37d6194c5bfe415337"), ObjectId("5b3a390fd6194c5bfe402699"), ObjectId("5b3a399cd6194c5bfe406793"), ObjectId("5b3a373cd6194c5bfe3f5565"), ObjectId("5b3a346ad6194c5bfe3df339"), ObjectId("5b3a3584d6194c5bfe3e718b"), ObjectId("5b3a37e5d6194c5bfe3f97d7") ], "count" : 8 }
{ "_id" : { "no" : 128015326 }, "uniqueIds" : [ ObjectId("5b3a3ed1d6194c5bfe42f1a3"), ObjectId("5b3a3df4d6194c5bfe4291f9"), ObjectId("5b3a38bed6194c5bfe400437"), ObjectId("5b3a381dd6194c5bfe3fd124"), ObjectId("5b3a3465d6194c5bfe3df1a7"), ObjectId("5b3a36cdd6194c5bfe3f2569"), ObjectId("5b3a3719d6194c5bfe3f4637"), ObjectId("5b3a377bd6194c5bfe3f7dfa") ], "count" : 8 }
{ "_id" : { "no" : 123932233 }, "uniqueIds" : [ ObjectId("5b3a3df3d6194c5bfe429177"), ObjectId("5b3a3d1dd6194c5bfe4224be"), ObjectId("5b3a3c39d6194c5bfe41b681"), ObjectId("5b3a3c0fd6194c5bfe41ac0d"), ObjectId("5b3a34e2d6194c5bfe3e22e6"), ObjectId("5b3a349fd6194c5bfe3e077d"), ObjectId("5b3a35aed6194c5bfe3e8b86"), ObjectId("5b3a3886d6194c5bfe3fea0f") ], "count" : 8 }
{ "_id" : { "no" : 150567857 }, "uniqueIds" : [ ObjectId("5b3a3a97d6194c5bfe413812"), ObjectId("5b3a39d9d6194c5bfe408888"), ObjectId("5b3a3816d6194c5bfe3fcd1b"), ObjectId("5b3a3635d6194c5bfe3ef80f"), ObjectId("5b3a3480d6194c5bfe3e011d"), ObjectId("5b3a34ead6194c5bfe3e2661"), ObjectId("5b3a33f8d6194c5bfe3dc477"), ObjectId("5b3a3529d6194c5bfe3e616a") ], "count" : 8 }
{ "_id" : { "no" : 143815726 }, "uniqueIds" : [ ObjectId("5b3a3b46d6194c5bfe415985"), ObjectId("5b3a3a08d6194c5bfe409172"), ObjectId("5b3a39a5d6194c5bfe4069fd"), ObjectId("5b3a33dad6194c5bfe3db527"), ObjectId("5b3a399cd6194c5bfe4067e7"), ObjectId("5b3a348fd6194c5bfe3e03a3"), ObjectId("5b3a37bcd6194c5bfe3f89c6"), ObjectId("5b3a36b7d6194c5bfe3f1cb9") ], "count" : 8 }
{ "_id" : { "no" : 143839331 }, "uniqueIds" : [ ObjectId("5b3a3ee8d6194c5bfe42f43a"), ObjectId("5b3a3e6ed6194c5bfe42bbbd"), ObjectId("5b3a3e04d6194c5bfe429773"), ObjectId("5b3a38c7d6194c5bfe40052f"), ObjectId("5b3a3a07d6194c5bfe40912e"), ObjectId("5b3a38f5d6194c5bfe401b19"), ObjectId("5b3a3b99d6194c5bfe417eb0"), ObjectId("5b3a39a5d6194c5bfe4069e8") ], "count" : 8 }
{ "_id" : { "no" : 99764925 }, "uniqueIds" : [ ObjectId("5b3a39f7d6194c5bfe408db1"), ObjectId("5b3a39c0d6194c5bfe4075eb"), ObjectId("5b3a399cd6194c5bfe40678a"), ObjectId("5b3a37bcd6194c5bfe3f89bf"), ObjectId("5b3a368ad6194c5bfe3f0673"), ObjectId("5b3a348fd6194c5bfe3e0391"), ObjectId("5b3a3578d6194c5bfe3e6e39"), ObjectId("5b3a36b7d6194c5bfe3f1ca8") ], "count" : 8 }

Вопрос: Какой самый простой и быстрый способ удалить дубликаты в Mongo 3.6.5 ? Как мне отредактировать вышеуказанную команду, чтобы добиться этого?

Я пробовал следующее, но оно не работает: 

db.books.aggregate([ { $group:{ _id: {no: "$no" }, uniqueIds: { $addToSet: "$_id" }, count: {$sum: 1} } }, { $match: {count: {$gt: 1} } } ]).forEach(function(doc){ doc.uniqueIds.shift(); db.uniqueIds.remove({_id : {$in: doc.uniqueIds}});})

1 Ответ

0 голосов
/ 24 июня 2019

Mongo-3.6: Здесь ReviewId - мой уникальный ключ, хочу удалить мое дублирующее значение reviewId 

cursor=db.reviewsAnalysed.aggregate([{"$group": {"_id": {"reviewId":"$reviewId"}, 
"unique_ids": {"$addToSet": "$_id"}, "count": {"$sum": 1 } } }])
response = []
for doc in cursor:
    del doc["unique_ids"][0]
    for id in doc["unique_ids"]:
        response.append(id)
db.reviewsAnalysed.remove({"_id": {"$in": response}})
...