[{"time":1,"holdings":[{"owner":"A","tokens":2},{"owner":"B","tokens":1}]},{"time":2,"holdings":[{"owner":"B","tokens":2}]},{"time":3,"holdings":[{"owner":"A","tokens":3},{"owner":"B","tokens":1},{"owner":"C","tokens":1}]},{"time":4,"holdings":[{"owner":"C","tokens":0}]}]

db.collection.aggregate([{$unwind:"$holdings"},{$group:{_id:"$holdings.owner",times:{$push:{time:"$time",tokens:"$holdings.tokens"}},minTime:{$min:"$time"}}},{$addFields:{times:{$reduce:{input:{$range:["$minTime",5]},initialValue:{values:[],lastIndex:0},in:{values:{"$concatArrays":["$$value.values",[{$cond:[{$in:["$$this","$times.time"]},{"$arrayElemAt":["$times","$$value.lastIndex"]},{"$mergeObjects":[{tokens:0},{"$arrayElemAt":["$times",{$subtract:["$$value.lastIndex",1]}]},{time:"$$this"}]}]}]]},lastIndex:{$cond:[{$in:["$$this","$times.time"]},{$sum:["$$value.lastIndex",1]},"$$value.lastIndex"]}}}}}},{$unwind:"$times.values"},{$group:{_id:"$times.values.time",holdings:{$push:{owner:"$_id",tokens:"$times.values.tokens"}}}},{$project:{_id:0,time:"$_id",holdings:1}},{$sort:{time:1}}])