[{"time":1,"holdings":[{"owner":"A","tokens":2},{"owner":"B","tokens":1}]},{"time":2,"holdings":[{"owner":"B","tokens":2}]},{"time":3,"holdings":[{"owner":"A","tokens":3},{"owner":"B","tokens":1},{"owner":"C","tokens":1}]},{"time":4,"holdings":[{"owner":"C","tokens":0}]}]

db.collection.aggregate([{$setWindowFields:{sortBy:{time:1},output:{allTimes:{"$addToSet":"$time",window:{documents:["unbounded","current"]}}}}},{$setWindowFields:{sortBy:{time:-1},output:{allTimes:{"$addToSet":"$allTimes",window:{documents:["unbounded","current"]}}}}},{$set:{allTimes:{$reduce:{input:"$allTimes",initialValue:[],in:{"$concatArrays":["$$value","$$this"]}}}}},{$set:{allTimes:{$setIntersection:"$allTimes"}}},{$unwind:"$holdings"},{$sort:{time:1}},{$group:{_id:"$holdings.owner",tokens:{$push:{tokens:"$holdings.tokens",time:"$time"}},times:{$push:"$time"},firstTime:{$first:"$time"},allTimes:{$first:"$allTimes"}}},{"$addFields":{"wantedTimes":{$filter:{input:"$allTimes",as:"item",cond:{$gte:["$$item","$firstTime"]}}}}},{$project:{tokens:1,wantedTimes:{"$setDifference":["$wantedTimes","$times"]}}},{$set:{data:{"$map":{"input":"$wantedTimes",as:"item","in":{time:"$$item",tokens:null}}}}},{$project:{tokens:{"$concatArrays":["$tokens","$data"]}}},{$unwind:"$tokens"},{$setWindowFields:{partitionBy:"$_id",sortBy:{"tokens.time":1},output:{lastTokens:{"$push":"$tokens.tokens",window:{documents:["unbounded","current"]}}}}},{$set:{lastTokens:{$filter:{input:"$lastTokens",as:"item",cond:{$ne:["$$item",null]}}}}},{$set:{"tokens.tokens":{$ifNull:["$tokens.tokens",{$last:"$lastTokens"}]}}},{$group:{_id:"$tokens.time",holdings:{$push:{owner:"$_id",tokens:"$tokens.tokens"}}}},{$project:{time:"$_id",holdings:1,_id:0}},{$sort:{time:1}}])