[{from:"user1",to:"user2"},{from:"user1",to:"user2"},{from:"user1",to:"user2"},{from:"user2",to:"user1"},{from:"user3",to:"user1"}]

db.collection.aggregate([{$match:{$or:[{to:"user1"},{from:"user1"}]}},{$group:{_id:null,from:{$addToSet:"$from"},to:{$addToSet:"$to"}}},{$project:{_id:0,users:{$filter:{input:{$setUnion:["$from","$to"]},cond:{$ne:["$$this","user1"]}}}}}])