What is the right way to add a sized PageView in Flutter?

Something that you are missing is to convert the globalPosition into localPosition with respect to the RenderBox. You can achieve it like

// onDragUpdate with the Painting Context
RenderBox referenceBox = context.findRenderObject();
Offset localPosition = referenceBox.globalToLocal(details.globalPosition);
// then use the localPosition to draw

Example usage for your use case as in here:

class _PainterRenderBox extends RenderBox {
  final _lines = new List<Path>();
  PanGestureRecognizer _drag;
  Path _currentPath;

  // variable to store padding
  Offset padding;

  _PainterRenderBox() {
    final GestureArenaTeam team = new GestureArenaTeam();
    _drag = new PanGestureRecognizer()
      ..team = team
      ..onStart = _handleDragStart
      ..onUpdate = _handleDragUpdate
      ..onEnd = _handleDragEnd;
  }

  @override
  bool get sizedByParent => true;

  @override
  bool hitTestSelf(Offset position) => true;

  @override
  handleEvent(PointerEvent event, BoxHitTestEntry entry) {
    assert(debugHandleEvent(event, entry));
    if (event is PointerDownEvent) {
      _drag.addPointer(event);
    }
  }

  @override
  paint(PaintingContext context, Offset offset) {
    final Canvas canvas = context.canvas;

    // update padding
    padding = offset;

    final Paint paintBorder = new Paint()
      ..strokeWidth = 1.0
      ..style = PaintingStyle.stroke
      ..color = Colors.white.withAlpha(128);
    canvas.drawRect(offset & size, paintBorder);

    final Paint paintPath = new Paint()
      ..strokeWidth = 5.0
      ..style = PaintingStyle.stroke
      ..color = Colors.white;
    _lines.forEach((path) {
      canvas.drawPath(path, paintPath);
    });
  }

  // check if the point lies inside drawable area
  bool _canDraw(Offset offset){
    return (padding & size).contains(offset);
  }

  void _handleDragStart(DragStartDetails details) {
    _currentPath = new Path();
    Offset point = globalToLocal(details.globalPosition); // convert globalPosition to localPosition
    point = padding + point; // add the padding to localPosition if any
    // check if point lies inside drawable area and then markNeedsPaint
    if(_canDraw(point)){
      _currentPath?.moveTo(point.dx, point.dy);
      _lines.add(_currentPath);
      markNeedsPaint();
    }
  }

  void _handleDragUpdate(DragUpdateDetails details) {
    Offset point = globalToLocal(details.globalPosition); // convert globalPosition to localPosition
    point = padding + point; // add the padding to localPosition if any
    // check if point lies inside drawable area and then markNeedsPaint
    if(_canDraw(point)){
      _currentPath?.lineTo(point.dx, point.dy);
      markNeedsPaint();
    }
  }

  void _handleDragEnd(DragEndDetails details) {
    _currentPath = null;
    markNeedsPaint();
  }
}

There is a question with a similar use case which allows the user to sign on the screen. Hope that might help you to know how to keep track of path. You can take a look at it here.

Hope that helps!