Database

db={"chats":[{"toUser":123,"fromUser":456,"message":"1 from suresh","timeStamp":"2019-10-09 16:39:14:1414 PM +05:30"},{"toUser":456,"fromUser":123,"message":"Man super man ","timeStamp":"2019-10-09 16:43:09:0909 PM +05:30"},{"toUser":456,"fromUser":123,"message":"Kk","timeStamp":"2019-10-09 18:31:12:1212 PM +05:30"},{"toUser":456,"fromUser":123,"message":"It working man","timeStamp":"2019-10-09 18:31:18:1818 PM +05:30"},{"toUser":456,"fromUser":123,"message":"2","timeStamp":"2019-10-09 18:31:21:2121 PM +05:30"},{"toUser":101,"fromUser":123,"message":"Kk","timeStamp":"2019-10-09 18:31:12:1212 PM +05:30"}]}

Query

db.chats.aggregate([{$match:{$or:[{"toUser":123},{"fromUser":123}]}},{"$project":{toUser:1,fromUser:1,message:1,timeStamp:1,fromToUser:["$fromUser","$toUser"]}},{$unwind:"$fromToUser"},{$sort:{"fromToUser":1}},{$group:{_id:"$_id","fromToUser":{$push:"$fromToUser"},"fromUser":{"$first":"$fromUser"},"toUser":{"$first":"$toUser"},"message":{"$first":"$message"},"timeStamp":{"$first":"$timeStamp"}}},{"$sort":{"timeStamp":-1}},{"$group":{"_id":"$fromToUser","fromUser":{"$first":"$fromUser"},"toUser":{"$first":"$toUser"},"message":{"$first":"$message"},"timeStamp":{"$first":"$timeStamp"}}}])

Result