You should file an enhancement request Radar for this behavior if you think it should be provided by the framework in some way.
But to workaround this in the meantime, you can create your own custom NSStoryboardSegue subclass to help with this.
@interface TablePopoverSegue : NSStoryboardSegue
@property (weak) NSTableView *anchorTableView;
@property NSRectEdge preferredEdge;
@property NSPopoverBehavior popoverBehavior;
@end
@implementation TablePopoverSegue
- (void)perform {
if ([self anchorTableView]) {
NSInteger selectedColumn = [[self anchorTableView] selectedColumn];
NSInteger selectedRow = [[self anchorTableView] selectedRow];
// If we can pick a specific row to show from, do that; otherwise just fallback to showing from the tableView
NSView *anchorView = [self anchorTableView];
if (selectedRow >= 0) {
anchorView = [[self anchorTableView] viewAtColumn:selectedColumn row:selectedRow makeIfNecessary:NO];
}
// Use the presentation API so that the popover can be dismissed using -dismissController:.
[[self sourceController] presentViewController:[self destinationController] asPopoverRelativeToRect:[anchorView bounds] ofView:anchorView preferredEdge:[self preferredEdge] behavior:[self popoverBehavior]];
}
}
@end
This can be specified in IB in the inspector panel for the segue (just like iOS):

And then in your source view controller's prepareForSegue:, you can just set up the segue:
- (void)prepareForSegue:(NSStoryboardSegue *)segue sender:(id)sender {
if ([segue isKindOfClass:[TablePopoverSegue class]]) {
TablePopoverSegue *popoverSegue = (TablePopoverSegue *)segue;
popoverSegue.preferredEdge = NSMaxXEdge;
popoverSegue.popoverBehavior = NSPopoverBehaviorTransient;
popoverSegue.anchorTableView = [self tableView];
}
}