A tutorial (mostly just commented code) to make a pixel fishing bot

[GlittPrizes]

Make sure to read all the comment lines, or it likely won't work for you out of the box. I will be showing the key concepts using Windows Forms. The first thing to do is setup a transparent overlay form that matches the game window. To add an additional form, right click the solution and add a new form.

Overlay.cs

public partial class Overlay : Form
{
    private RECT _rect;
    private int _initialStyle;

    private readonly Pen _framePen = new Pen(Color.CadetBlue);
    private bool _mouseDraw;

    public struct RECT
    {
        public int Left, Top, Right, Bottom;
    }

    public Overlay()
    {
        InitializeComponent();
    }

    // Actions taken when the form loads
    private void Overlay_Load(object sender, EventArgs e)
    {

        // prevent flickering by enabling this
        DoubleBuffered = true;
        
        // use a color key to turn everything transparent
        BackColor = Color.Wheat;
        TransparencyKey = Color.Wheat;

        TopMost = true;
        // hide the title bar and window outline
        FormBorderStyle = FormBorderStyle.None;
        // store the original window focus properties and make the window function as click-through
        _initialStyle = GetWindowLong(Handle, -20);
        ChangeFocus();

        // make the overlay the same size and location as the game
        GetWindowRect(WoWProcess.Handle, out _rect);
        Size = new Size(_rect.Right - _rect.Left, _rect.Bottom - _rect.Top);
        Top = _rect.Top;
        Left = _rect.Left;
    }

    // toggle the ability to click the overlay for selecting the pixel search area
    public void ChangeFocus(bool active = false)
    {
        if (active)
        {
            _mouseDraw = true;
            SetWindowLong(Handle, -20, _initialStyle);
        }
        else
        {
            SetWindowLong(Handle, -20, _initialStyle | 0x80000 | 0x20);
        }
    }

    protected override void OnPaint(PaintEventArgs e)
    {
        if (!_mouseDraw) return;

        e.Graphics.SmoothingMode = SmoothingMode.HighQuality;

        var mousePos = PointToClient(MousePosition);
        var selectedRect = new Rectangle(Math.Min(MouseHelper.DownPoint.X, mousePos.X), Math.Min(MouseHelper.DownPoint.Y, mousePos.Y), Math.Abs(mousePos.X - MouseHelper.DownPoint.X),  Math.Abs(mousePos.Y - MouseHelper.DownPoint.Y));
        e.Graphics.DrawRectangle(_framePen, selectedRect);
    }

    protected override void OnResize(EventArgs e)
    {
        base.OnResize(e);
        Invalidate();
    }

    protected override void OnMouseDown(MouseEventArgs e)
    {
        base.OnMouseDown(e);

        if (e.Button == MouseButtons.Left)
            MouseHelper.DownPoint = e.Location;
    }

    protected override void OnMouseMove(MouseEventArgs e)
    {
        base.OnMouseMove(e);
        if (e.Button == MouseButtons.Left && MouseHelper.DownPoint != Point.Empty)
            Invalidate();
    }

    protected override void OnMouseUp(MouseEventArgs e)
    {
        base.OnMouseUp(e);
        
        _mouseDraw = false;
        MouseHelper.UpPoint = e.Location;

        ChangeFocus();

        Invalidate();
    }

    [DllImport("user32.dll", SetLastError = true)]
    private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);

    [DllImport("user32.dll")]
    private static extern int SetWindowLong(IntPtr hWnd, int nIndex, int dwNewLong);

    [DllImport("user32.dll", SetLastError = true)]
    private static extern int GetWindowLong(IntPtr hWnd, int nIndex);

Some parts of the program need access to the window handle.

WoWProcess.cs

public static class WoWProcess
{
    public const string ProcessName = "Wow";
    public static IntPtr Handle = System.Diagnostics.Process.GetProcessesByName(ProcessName)[0].MainWindowHandle;
}

The main form is something like this. It's just a button to start and a button to define the pixel search area. To define the search area, click "Define Search" then drag a box inside the game.

Form1.cs

public partial class AnglesForm : Form
{
    private CancellationTokenSource _cancellationToken;

    private readonly Overlay _overlay = new Overlay();
    private readonly GDI _gdi = new GDI();

    private bool _isFishing;

    public AnglesForm()
    {
        InitializeComponent();
    }

    private void AnglesForm_Load(object sender, EventArgs e)
    {
        startButton.BackColor = Color.PowderBlue;

        _overlay.Show();

        Window.Init();
    }

    // make sure the click event is set on your form's start button
    private void startButton_Click(object sender, EventArgs e)
    {
        if (!_isFishing)
        {
            _isFishing = true;

            startButton.Text = @"Stop";
            startButton.BackColor = Color.LawnGreen;

            StartScan();
        }
        else
        {
            _isFishing = false;

            startButton.Text = @"Start";
            startButton.BackColor = Color.PowderBlue;

            UpdateStatus(@"idle");

            StopScan();
        }
    }

    private void searchWindowButton_Click(object sender, EventArgs e)
    {
        _overlay.ChangeFocus(true);
    }

    // invoke is used to update the gui across other threads
    private void UpdateStatus(string msg)
    {
        var output = "Status: " + msg;
        statusLabel.Invoke((MethodInvoker)delegate { statusLabel.Text = output; });
    }

    private void StartScan()
    {
        _cancellationToken = new CancellationTokenSource();

        Task.Factory.StartNew(DoScan, _cancellationToken.Token);
    }

    // main scanning loop
    private void DoScan()
    {
        while (!_cancellationToken.IsCancellationRequested)
        {
            UpdateStatus(@"scanning");

            _gdi.SetClosestColor();

            if (ColorWithinThreshold())
                break;
        }

        StopScan();
        LootFish();
    }

    private void StopScan()
    {
        _cancellationToken?.Cancel();
    }

    // compares the brightest point to a threshold value - might need a night mode or adjustments for differently lighted areas of the game
    private bool ColorWithinThreshold()
    {
        const float minThreshold = 0.23f;

        return GDI.ColorThreshold.LastBrightness < minThreshold;
    }

    private void LootFish()
    {
        UpdateStatus(@"fish caught!");

        _gdi.FindBobber();

       // x value sometimes needs offset, I'm not sure what is off about it
        var mouseDestination = new Point(GDI.ColorThreshold.ColorX + Window.Dimensions.X + MouseHelper.DownPoint.X, GDI.ColorThreshold.ColorY + Window.Dimensions.Y + MouseHelper.DownPoint.Y);
        var mouseAwayPoint = new Point(200, 300);

        MouseHelper.Move(mouseDestination);
        Thread.Sleep(200);
        MouseHelper.RightClick();
        Thread.Sleep(200);
        MouseHelper.Move(mouseAwayPoint);

        GDI.ColorThreshold.LastBrightness = 1f;
        GDI.ColorThreshold.LastColor = Color.Black;
        GDI.ColorThreshold.LastHue = 200f;

        Thread.Sleep(200);
        StartScan();

        UpdateStatus(@"waiting");
    }
}

The mouse movement is near instant. I didn't bother with any humanizing.

MouseHelper.cs

public static class MouseHelper
{
    public enum Vk : int
    {
        VkDown = 0x28,
        VkUp = 0x26,
        VkLeft = 0x25,
        VkRight = 0x27
    }

    public enum Wm
    {
        WmKeyup = 0x0101,
        WmKeydown = 0x0100
    }

    public static Point DownPoint = Point.Empty;
    public static Point UpPoint = Point.Empty;

    public static void Move(Point position)
    {
        Cursor.Position = position;
    }

    private static void RightDown()
    {
        SendMessage(WoWProcess.Handle, 0x200, (IntPtr)0, (IntPtr)0);
        SendMessage(WoWProcess.Handle, 0x204, (IntPtr)0, (IntPtr)0);
    }

    private static void RightUp()
    {
        SendMessage(WoWProcess.Handle, 0x200, (IntPtr)0, (IntPtr)0);
        SendMessage(WoWProcess.Handle, 0x205, (IntPtr)0, (IntPtr)0);
    }

    public static void RightClick()
    {
        RightDown();
        RightUp();
    }

    [DllImport("user32.dll", CharSet = CharSet.Unicode)]
    public static extern int SendMessage(IntPtr hWnd, UInt32 Msg, IntPtr wParam, IntPtr lParam);

    [DllImport("user32.dll")]
    private static extern bool SetCursorPos(int x, int y);
}

GDI.cs is the main pixel scanning code. This example uses PixelMap, so you'll have to install that Nuget package.

GDI.cs

public class GDI
{
    private enum RasterOp : uint
    {
        /// <summary>dest = source</summary>
        SRCCOPY = 0x00CC0020,
        /// <summary>dest = source OR dest</summary>
        SRCPAINT = 0x00EE0086,
        /// <summary>dest = source AND dest</summary>
        SRCAND = 0x008800C6,
        /// <summary>dest = source XOR dest</summary>
        SRCINVERT = 0x00660046,
        /// <summary>dest = source AND (NOT dest)</summary>
        SRCERASE = 0x00440328,
        /// <summary>dest = (NOT source)</summary>
        NOTSRCCOPY = 0x00330008,
        /// <summary>dest = (NOT src) AND (NOT dest)</summary>
        NOTSRCERASE = 0x001100A6,
        /// <summary>dest = (source AND pattern)</summary>
        MERGECOPY = 0x00C000CA,
        /// <summary>dest = (NOT source) OR dest</summary>
        MERGEPAINT = 0x00BB0226,
        /// <summary>dest = pattern</summary>
        PATCOPY = 0x00F00021,
        /// <summary>dest = DPSnoo</summary>
        PATPAINT = 0x00FB0A09,
        /// <summary>dest = pattern XOR dest</summary>
        PATINVERT = 0x005A0049,
        /// <summary>dest = (NOT dest)</summary>
        DSTINVERT = 0x00550009,
        /// <summary>dest = BLACK</summary>
        BLACKNESS = 0x00000042,
        /// <summary>dest = WHITE</summary>
        WHITENESS = 0x00FF0062,
        /// <summary>
        /// Capture window as seen on screen.  This includes layered windows
        /// such as WPF windows with AllowsTransparency="true"
        /// </summary>
        CAPTUREBLT = 0x40000000
    }

    public static class ColorThreshold
    {
        public static Color LastColor = Color.Black;
        public static float LastBrightness = 1f;
        public static float LastHue = 200f;
        public static int ColorX;
        public static int ColorY;
    }

    public void SetClosestColor()
    {
        var graphics = SetGraphics(out var pixelMap);

        SearchPixels(graphics, pixelMap, Color.White);
    }

    public void FindBobber()
    {
        var graphics = SetGraphics(out var pixelMap);

        SearchPixels(graphics, pixelMap, Color.FromArgb(86,25,20), true);
    }

    private static Graphics SetGraphics(out PixelMap pixelMap)
    {
        var x = MouseHelper.DownPoint.X;
        var y = MouseHelper.DownPoint.Y;
        var width = MouseHelper.UpPoint.X - x;
        var height = MouseHelper.UpPoint.Y - y;

        var rect = new Rectangle(x, y, width, height);

        var captureBitmap = new Bitmap(width, height, PixelFormat.Format24bppRgb);

        var graphics = Graphics.FromImage(captureBitmap);
        graphics.SetClip(rect);

        graphics.CopyFromScreen(rect.Left + Window.Dimensions.X, rect.Top + Window.Dimensions.Y, 0, 0, new Size(width, height), CopyPixelOperation.SourceCopy);

        pixelMap = new PixelMap(captureBitmap);
        return graphics;
    }

    private static void SearchPixels(Graphics graphics, PixelMap pixelMap, Color matchColor, bool matchHue = false)
    {
        for (var y = 0; y < graphics.ClipBounds.Y; y++)
        {
            for (var x = 0; x < graphics.ClipBounds.X; x++)
            {
                var color = pixelMap[x, y].Color;
                var tmpColor = Color.FromArgb(255, color.R, color.G, color.B);

                CheckColorThreshold(matchColor, tmpColor, x, y, matchHue);
            }
        }
    }

    private static void CheckColorThreshold(Color matchColor, Color currentColor, int x, int y, bool matchHue = false)
    {
        var threshold = matchHue ? Math.Abs(matchColor.GetHue() - currentColor.GetHue()) : Math.Abs(matchColor.GetBrightness() - currentColor.GetBrightness());

        if(matchHue && threshold >= ColorThreshold.LastHue) return;
        else if (!matchHue && threshold >= ColorThreshold.LastBrightness) return;

        if (matchHue)
        {
            ColorThreshold.LastHue = threshold;
            ColorThreshold.ColorX = x;
            ColorThreshold.ColorY = y;
        }
        else
            ColorThreshold.LastBrightness = threshold;

        ColorThreshold.LastColor = currentColor;
    }

    [DllImport("gdi32.dll", EntryPoint = "BitBlt", SetLastError = true)]
    [return: MarshalAs(UnmanagedType.Bool)]
    private static extern bool BitBlt([In] IntPtr hdc, int nXDest, int nYDest, int nWidth, int nHeight, [In] IntPtr hdcSrc, int nXSrc, int nYSrc, RasterOp dwRop);

    [DllImport("user32.dll", SetLastError = false)]
    private static extern IntPtr GetDesktopWindow();

    [DllImport("user32.dll")]
    private static extern IntPtr GetWindowDC(IntPtr hWnd);
}

The game's dimensions are used in some calculations.

Window.cs

public static class Window
{
    public static Rectangle Dimensions { get; private set; }

    public static void Init()
    {
        Rectangle rect = default;
        GetWindowRect(WoWProcess.Handle, ref rect);
        Dimensions = rect;
    }

    [DllImport("user32.dll", SetLastError = true)]
    private static extern bool GetWindowRect(IntPtr hWnd, ref Rectangle lpRect);
}

[GlittPrizes]

I've been sharing ideas with a friend who's a few steps ahead of me in the fishing scene and the best approach seems to be a balance of luminance scanning and an object recognition model. If you just rely on one of these, you'll spend a lot of time tweaking parameters or thresholds, but if you have them both then the object model can confirm the luminance result.

If you dig through github fish bots (Don't use them! they are all detected!) there are a few other techniques that can be done such as sound detection or color amount scanning. Likely the best method is to peek through the object manager and find when the bobber value changes to have 100% accuracy.

This thread kind of died. I hope at least someone found something useful. If anything the overlay method used is pretty generic/universal for other things if you aren't using ImGui already or whatnot.