MapBox Unity SDK : How to correctly calculate raycast position on a topdown 2D map in order to place POIs at runtime? Using "Zoomable Map" as a base

Question

This is my frist time posting here.

I'm using the Mapbox Unity SDK in order to aid the creation of an app I'm developing. At the base of it, we have a 2D top-down map in which we can drag and drop UI images into the map to turn them into geolocalised POIs. I'm using a modified SpawnOnMap script:

using UnityEngine;
    using Mapbox.Utils;
    using Mapbox.Unity.Map;
    using Mapbox.Unity.MeshGeneration.Factories;
    using Mapbox.Unity.Utilities;
    using System.Collections.Generic;

    public class SpawnOnMap : MonoBehaviour
    {
        [SerializeField]
        AbstractMap _map;

        [SerializeField]
        [Geocode]
        string[] _locationStrings;
        List<Vector2d> _locations;


        public float _spawnScale = 100f;


        public GameObject _markerPrefab;

        List<GameObject> _spawnedObjects;

        void Start()
        {
            _locations = new List<Vector2d>();
            _spawnedObjects = new List<GameObject>();
            for (int i = 0; i < _locationStrings.Length; i++)
            {
                var locationString = _locationStrings[i];
                _locations[i] = Conversions.StringToLatLon(locationString);
                var instance = Instantiate(_markerPrefab);
                instance.transform.localPosition = _map.GeoToWorldPosition(_locations[i], true);
                instance.transform.localScale = new Vector3(_spawnScale, _spawnScale, _spawnScale);
                _spawnedObjects.Add(instance);
            }
        }

        public void IncreaseMarkers(GameObject inst)
        {
            _spawnedObjects.Add(inst);
        }
        public void IncreaseLocations(Vector2d loc)
        {
            _locations.Add(loc);
        }
        private void Update()
        {
            int count = _spawnedObjects.Count;
            for (int i = 0; i < count; i++)
            {
                var spawnedObject = _spawnedObjects[i];
                var location = _locations[i];
                spawnedObject.transform.localPosition = _map.GeoToWorldPosition(location, true);
                spawnedObject.transform.localScale = new Vector3(_spawnScale, _spawnScale, _spawnScale);
            }
        }
    }

and a 'AddPOI' script which is attached to the image:

using UnityEngine;
using Mapbox.Utils;
using Mapbox.Unity.Utilities;
using Mapbox.Unity.Map;
using Mapbox.Examples;

public class AddPOI : MonoBehaviour
{
    float clicktime;
    Ray ray;
    RaycastHit hit;

    AbstractMap _map;
    [SerializeField]
    Transform clickPoint;
    [SerializeField]
    LayerMask layerMask;

    [SerializeField] private SpawnOnMap spawner;




    

    // Start is called before the first frame update
    void Start()
    {
        _map = FindObjectOfType<AbstractMap>();
    }

    void Update()
    {

        bool click = false;

        if (Input.GetMouseButtonDown(0))
        {
            clicktime = Time.time;
        }
        if (Input.GetMouseButtonUp(0))
        {
            if (Time.time - clicktime < 0.15f)
            {
                //click = true;
            }
        }

        if (click)
        {
            ray = Camera.main.ScreenPointToRay(Input.mousePosition);
            print(Input.mousePosition);
            if (Physics.Raycast(ray, out hit, Mathf.Infinity, layerMask))
            {               
                clickPoint.position = hit.point;
                var clickLocation = new Vector2d();
                clickLocation = clickPoint.GetGeoPosition(_map.CenterMercator, _map.WorldRelativeScale);
                print(clickLocation);
                AddPOIOnLocation(clickLocation);
            }
        }
    }

    public Vector2d GetLocation()
    {
        ray = Camera.main.ScreenPointToRay(Input.mousePosition);

        if (Physics.Raycast(ray, out hit, Mathf.Infinity, layerMask))
        {
            clickPoint.position = hit.point;
            Vector2d clickLocation = clickPoint.GetGeoPosition(_map.CenterMercator, _map.WorldRelativeScale);
            print(clickLocation.x + ", " + clickLocation.y);
            return clickLocation;
        }
        else return Vector2d.zero;
    }

    public void AddPOIOnLocation(Vector2d location)
    {       
        var instance = Instantiate(spawner._markerPrefab);
        instance.transform.localPosition = _map.GeoToWorldPosition(location, true);
        instance.transform.localScale = new Vector3(spawner._spawnScale, spawner._spawnScale, spawner._spawnScale);
        spawner.IncreaseMarkers(instance);
        spawner.IncreaseLocations(location);
    }
}

I'm also going from the ZoomableMap example, thus using their pan and zoom script. Problem is, my script works fine at an angled rotation (the same one present in nearly all other examples, X 35.7 and Y 12.513), but, as soon as I change those angles, the map or camera position, things go haywire. I managed to make it work at a single zoom level and a modified camera position, but otherwise the positions registered in the raycast do not reflect the mouse pointer on the map. The further I go from the original angles, the worse it gets. At one point I just get an insane number that even gets unity to complain about floating point innacuracies. What gives?

Any help is greatly appreciated.