Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/ui-automation.md
Original file line number Diff line number Diff line change
Expand Up @@ -747,6 +747,7 @@ winapp ui touch tile-photo-7b3c -a myapp --gesture long-press --hold-ms 600
winapp ui touch -a myapp --at 100,300 --gesture swipe --to-point 400,300
winapp ui touch img-map-9f8e -a myapp --gesture pinch --distance 200 # pinch-to-zoom out (2 fingers)
winapp ui touch img-map-9f8e -a myapp --gesture stretch --distance 200 # stretch-to-zoom in (2 fingers)
winapp ui touch img-map-9f8e -a myapp --gesture pinch --distance 300 --angle 45 # diagonal pinch
```

**Options:**
Expand All @@ -755,6 +756,7 @@ winapp ui touch img-map-9f8e -a myapp --gesture stretch --distance 200 # stretc
- `--to-point <x,y>` — End point for a `swipe`. Takes precedence over `--direction`.
- `--direction <right|left|up|down>` — Swipe direction (default: `right`). Combined with `--distance` to compute the end point when `--to-point` is not given.
- `--distance <px>` — Finger spread for `pinch`/`stretch`, or swipe distance in pixels.
- `--angle <degrees>` — Finger axis for `pinch`/`stretch`, 0–359, counter-clockwise: `0` is horizontal (default), `90` is vertical, `45` runs from lower-left to upper-right.
- `--hold-ms <ms>` — Hold contacts down before lifting (long-press hold time; defaults to 500 ms for `long-press` when not set).
- `--duration-ms <ms>` — Glide time for moving gestures (swipe/pinch/stretch; default 300).
- `--fingers <n>` — Number of contacts (1–10; default 1). `pinch`/`stretch` always use 2.
Expand Down
4 changes: 2 additions & 2 deletions plugins/winapp/skills/winapp-ui-automation/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -322,9 +322,9 @@ winapp ui touch -a myapp --at 320,240
winapp ui touch tile-photo-7b3c -a myapp --gesture long-press --hold-ms 600
winapp ui touch -a myapp --at 100,300 --gesture swipe --to-point 400,300
winapp ui touch img-map-9f8e -a myapp --gesture pinch --distance 200
winapp ui touch img-map-9f8e -a myapp --gesture stretch --distance 200
winapp ui touch img-map-9f8e -a myapp --gesture stretch --distance 200 --angle 90
```
- Gestures: `tap` (default), `double-tap`, `long-press`, `swipe`, `pinch`, `stretch`. `--fingers` 1–10 (pinch/stretch always 2). Refuses without a non-zero foregrounded target (`no_target`/`foreground_not_target`/`no_interactive_desktop`); every coordinate is checked against the target window, and a point outside it produces a non-fatal warning (`warnings[]` in `--json`) while injection still proceeds — consistent with the mouse verbs. If injected touch is unsupported on the device, the command surfaces the real Win32 error rather than a false success. In a Remote Desktop/VM session delivery isn't guaranteed even on exit 0 — the command appends a delivery-uncertainty warning (`warnings[]` in `--json`); verify the effect with a screenshot.
- Gestures: `tap` (default), `double-tap`, `long-press`, `swipe`, `pinch`, `stretch`. `--fingers` 1–10 (pinch/stretch always 2). `--angle` 0–359 sets the pinch/stretch finger axis (0 horizontal, 90 vertical). Refuses without a non-zero foregrounded target (`no_target`/`foreground_not_target`/`no_interactive_desktop`); every coordinate is checked against the target window, and a point outside it produces a non-fatal warning (`warnings[]` in `--json`) while injection still proceeds — consistent with the mouse verbs. If injected touch is unsupported on the device, the command surfaces the real Win32 error rather than a false success. In a Remote Desktop/VM session delivery isn't guaranteed even on exit 0 — the command appends a delivery-uncertainty warning (`warnings[]` in `--json`); verify the effect with a screenshot.

### Pen / stylus (taps and ink strokes)
Inject synthetic pen input (Windows 10 1809+). Target an element center, an explicit `--at`, or a full `--path` ink stroke, with pressure/tilt/eraser control.
Expand Down
39 changes: 39 additions & 0 deletions src/winapp-CLI/WinApp.Cli.Tests/UiCommandTests.Touch.cs
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,10 @@ public async Task Touch_DelayAboveMax_RejectedWithInvalidArguments(string option
[DataRow("long-press", "--duration-ms", "50")]
[DataRow("pinch", "--fingers", "3")]
[DataRow("stretch", "--fingers", "3")]
[DataRow("tap", "--angle", "90")]
[DataRow("double-tap", "--angle", "90")]
[DataRow("long-press", "--angle", "90")]
[DataRow("swipe", "--angle", "0")]
public async Task Touch_IncompatibleExplicitGestureOption_RejectedWithInvalidArguments(
string gesture, string option, string value)
{
Expand Down Expand Up @@ -341,6 +345,41 @@ public async Task Touch_Pinch_ExplicitTwoFingers_Succeeds()
Assert.AreEqual(250, call.DurationMs);
}

[TestMethod]
public async Task Touch_Stretch_Angle90_MovesFingersVertically()
{
_fakeTargetResolver.TargetResult.WindowHandle = 5165;

var command = GetRequiredService<UiTouchCommand>();
var exitCode = await ParseAndInvokeWithCaptureAsync(command,
["-a", "TestApp", "--gesture", "stretch", "--at", "100,100",
"--distance", "80", "--angle", "90", "--json"]);

Assert.AreEqual(0, exitCode);
var call = _fakePointer.TouchCalls[0];
Assert.AreEqual(TouchGesture.Stretch, call.Gesture);
Assert.AreEqual(new PointerPoint(100, 104), call.ContactPaths[0][0]);
Assert.AreEqual(new PointerPoint(100, 140), call.ContactPaths[0][^1]);
Assert.AreEqual(new PointerPoint(100, 96), call.ContactPaths[1][0]);
Assert.AreEqual(new PointerPoint(100, 60), call.ContactPaths[1][^1]);
}

[TestMethod]
[DataRow("-1")]
[DataRow("360")]
public async Task Touch_Pinch_AngleOutOfRange_RejectedWithInvalidArguments(string angle)
{
var command = GetRequiredService<UiTouchCommand>();
var exitCode = await ParseAndInvokeWithCaptureAsync(command,
["-a", "TestApp", "--gesture", "pinch", "--at", "100,100",
"--distance", "80", "--angle", angle, "--json"]);

Assert.AreEqual(1, exitCode);
Assert.AreEqual(0, _fakePointer.TouchCalls.Count);
AssertJsonErrorCode(UiJsonError.CodeInvalidArguments);
StringAssert.Contains(ConsoleStdErr.ToString(), "--angle must be between 0 and 359");
}

[TestMethod]
public async Task Touch_ExplicitPointOutsideWindow_WarnsAndInjects()
{
Expand Down
22 changes: 21 additions & 1 deletion src/winapp-CLI/WinApp.Cli/Commands/UiTouchCommand.cs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,12 @@ internal class UiTouchCommand : Command, IShortDescription, IHelpExamples
DefaultValueFactory = _ => 1
};

public static Option<int> AngleOption { get; } = new("--angle")
{
Description = "Finger axis for pinch/stretch in degrees, 0-359, counter-clockwise: 0 = horizontal (default), 90 = vertical, 45 = lower-left to upper-right.",
DefaultValueFactory = _ => 0
};

public UiTouchCommand()
: base("touch", "Inject synthetic touch input using the Windows touch-injection API. " +
"Supports tap, double-tap, long-press, swipe, pinch and stretch gestures at an element's " +
Expand All @@ -103,6 +109,7 @@ public UiTouchCommand()
Options.Add(HoldOption);
Options.Add(DurationOption);
Options.Add(FingersOption);
Options.Add(AngleOption);
Options.Add(WinAppRootCommand.JsonOption);
UiQueryOptions.AddTo(this);
}
Expand Down Expand Up @@ -143,6 +150,8 @@ public class Handler(
bool directionWasSupplied = (parseResult.GetResult(DirectionOption)?.Tokens.Count ?? 0) > 0;
bool durationWasSupplied = (parseResult.GetResult(DurationOption)?.Tokens.Count ?? 0) > 0;
bool fingersWasSupplied = (parseResult.GetResult(FingersOption)?.Tokens.Count ?? 0) > 0;
var angle = parseResult.GetValue(AngleOption);
bool angleWasSupplied = (parseResult.GetResult(AngleOption)?.Tokens.Count ?? 0) > 0;

// All app-independent semantic validation runs BEFORE the missing-app check so that
// malformed argument values return invalid_arguments, not missing_app (M4 root-cause fix).
Expand Down Expand Up @@ -207,6 +216,16 @@ public class Handler(
return RejectInvalidArguments(parseResult, json, $"--duration-ms is only valid with moving gestures: swipe, pinch, or stretch (got {gestureStr}).");
}

if (angleWasSupplied && gesture is not (TouchGesture.Pinch or TouchGesture.Stretch))
{
return RejectInvalidArguments(parseResult, json, $"--angle is only valid with --gesture pinch or stretch (got {gestureStr}).");
}

if (angle is < 0 or > 359)
{
return RejectInvalidArguments(parseResult, json, $"--angle must be between 0 and 359 degrees. Got '{angle}'.");
}

// Long-press with no explicit --hold-ms defaults to 500 ms (a real long-press).
// An explicit --hold-ms 0 is a degenerate long-press (indistinguishable from a tap)
// and is rejected clearly rather than silently rewritten to 500.
Expand Down Expand Up @@ -317,6 +336,7 @@ protected override async Task<int> ExecuteAsync(ParseResult parseResult, IUiTurn
var holdMs = parseResult.GetValue(HoldOption);
var durationMs = parseResult.GetValue(DurationOption);
var fingers = parseResult.GetValue(FingersOption);
var angle = parseResult.GetValue(AngleOption);

_ = Gestures.TryGetValue(gestureStr, out var gesture);

Expand Down Expand Up @@ -376,7 +396,7 @@ protected override async Task<int> ExecuteAsync(ParseResult parseResult, IUiTurn
await PointerCommandSupport.SetForegroundAsync(desktopForeground, targetHwnd, cancellationToken);

(contactPaths, points, effectiveFingers) =
PointerGesturePlanner.PlanTouch(gesture, start, to, distance, fingers, direction);
PointerGesturePlanner.PlanTouch(gesture, start, to, distance, fingers, direction, angle);

prep = PointerCommandSupport.TryPrepareInjection(
uiAutomation, foregroundGuard, targetHwnd, points, "touch", "touch", logger, json);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,13 +100,13 @@ public void PlanTouch_Pinch_FingersStartApartAndConvergeTowardCenter()
public void PlanTouch_Pinch_StartCoordinatesRespectHalfDistance()
{
// The initial finger spread is distance/2 from center on each side (clamped to at least
// PinchCenterGapPx+1 per finger so fingers don't start inside each other).
// PinchCenterGapPx+2 per finger so rounding never collapses a finger's travel).
var start = new PointerPoint(200, 200);
int distance = 80; // half = 40
var (contactPaths, _, _) = PointerGesturePlanner.PlanTouch(
TouchGesture.Pinch, start, end: null, distance, fingers: 2);

int expectedHalf = Math.Max(5, distance / 2); // PinchCenterGapPx=4 → +1 = 5
int expectedHalf = Math.Max(6, distance / 2); // PinchCenterGapPx=4 → +2 = 6
Assert.AreEqual(start.X - expectedHalf, contactPaths[0][0].X,
"Left finger X start must be center.X - half");
Assert.AreEqual(start.X + expectedHalf, contactPaths[1][0].X,
Expand Down Expand Up @@ -173,6 +173,95 @@ public void PlanTouch_Stretch_IsReverseOfPinch()
"Right finger: stretch end must equal pinch start");
}

// -------------------------------------------------------------------------
// Pinch/stretch --angle
// -------------------------------------------------------------------------

[TestMethod]
public void PlanTouch_Pinch_AngleZero_MatchesDefaultHorizontalPaths()
{
var start = new PointerPoint(400, 300);
var (defaultPaths, _, _) = PointerGesturePlanner.PlanTouch(
TouchGesture.Pinch, start, end: null, distance: 100, fingers: 2);
var (anglePaths, _, _) = PointerGesturePlanner.PlanTouch(
TouchGesture.Pinch, start, end: null, distance: 100, fingers: 2, direction: null, angleDegrees: 0);

Assert.AreEqual(new PointerPoint(350, 300), defaultPaths[0][0]);
Assert.AreEqual(new PointerPoint(396, 300), defaultPaths[0][^1]);
for (int f = 0; f < 2; f++)
{
CollectionAssert.AreEqual(defaultPaths[f].ToList(), anglePaths[f].ToList(), $"Finger {f} must match the default");
}
}

[TestMethod]
[DataRow(90, 0, 50, 0, 4)] // vertical: first finger starts below center
[DataRow(45, -35, 35, -3, 3)] // "/" diagonal: first finger starts lower-left
[DataRow(135, 35, 35, 3, 3)] // "\" diagonal: first finger starts lower-right
public void PlanTouch_Pinch_Angle_RotatesFingerAxis(
int angle, int apartDx, int apartDy, int nearDx, int nearDy)
{
var start = new PointerPoint(400, 300);
var (paths, _, _) = PointerGesturePlanner.PlanTouch(
TouchGesture.Pinch, start, end: null, distance: 100, fingers: 2, direction: null, angleDegrees: angle);

Assert.AreEqual(new PointerPoint(400 + apartDx, 300 + apartDy), paths[0][0]);
Assert.AreEqual(new PointerPoint(400 + nearDx, 300 + nearDy), paths[0][^1]);
Assert.AreEqual(new PointerPoint(400 - apartDx, 300 - apartDy), paths[1][0]);
Assert.AreEqual(new PointerPoint(400 - nearDx, 300 - nearDy), paths[1][^1]);
}

[TestMethod]
[DataRow(1)]
[DataRow(9)]
[DataRow(300)]
public void PlanTouch_PinchAndStretch_EveryAngle_StaysSymmetricAndMoves(int distance)
{
// Rounding must never collapse a finger's start onto its end, break symmetry around the
// anchor, or make a pinch move outward — even at the smallest spread.
var start = new PointerPoint(400, 300);
for (int angle = 0; angle < 360; angle++)
{
var (pinch, _, _) = PointerGesturePlanner.PlanTouch(
TouchGesture.Pinch, start, end: null, distance, fingers: 2, direction: null, angleDegrees: angle);
var (stretch, _, _) = PointerGesturePlanner.PlanTouch(
TouchGesture.Stretch, start, end: null, distance, fingers: 2, direction: null, angleDegrees: angle);

for (int i = 0; i < 2; i++)
{
Assert.AreEqual(start.X * 2, pinch[0][i].X + pinch[1][i].X, $"angle {angle}: X not symmetric");
Assert.AreEqual(start.Y * 2, pinch[0][i].Y + pinch[1][i].Y, $"angle {angle}: Y not symmetric");
}

foreach (var path in pinch)
{
Assert.AreNotEqual(path[0], path[^1], $"angle {angle}: finger does not move");
Assert.IsTrue(DistanceSquared(path[0], start) > DistanceSquared(path[^1], start),
$"angle {angle}: pinch must move toward the anchor");
Assert.IsTrue(DistanceSquared(path[^1], start) > 0, $"angle {angle}: finger ends on the anchor");
}

for (int f = 0; f < 2; f++)
{
Assert.AreEqual(pinch[f][0], stretch[f][^1], $"angle {angle}: stretch must reverse pinch");
Assert.AreEqual(pinch[f][^1], stretch[f][0], $"angle {angle}: stretch must reverse pinch");
}
}

static int DistanceSquared(PointerPoint a, PointerPoint b)
=> (a.X - b.X) * (a.X - b.X) + (a.Y - b.Y) * (a.Y - b.Y);
}

[TestMethod]
public void PlanTouch_Swipe_IgnoresAngle()
{
var start = new PointerPoint(100, 100);
var (paths, _, _) = PointerGesturePlanner.PlanTouch(
TouchGesture.Swipe, start, end: null, distance: 50, fingers: 1, direction: "right", angleDegrees: 90);

Assert.AreEqual(new PointerPoint(150, 100), paths[0][^1]);
}

// -------------------------------------------------------------------------
// M9 — Finger-count coercion for pinch/stretch
// -------------------------------------------------------------------------
Expand Down
46 changes: 38 additions & 8 deletions src/winapp-CLI/WinApp.UIAutomation/Input/PointerGesturePlanner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,22 @@ public static (List<IReadOnlyList<PointerPoint>> ContactPaths, List<PointerPoint
int distance,
int fingers,
string? direction = null)
=> PlanTouch(gesture, start, end, distance, fingers, direction, angleDegrees: 0);

/// <summary>
/// Same as <see cref="PlanTouch(TouchGesture, PointerPoint, PointerPoint?, int, int, string?)"/>,
/// with <paramref name="angleDegrees"/> setting the pinch/stretch finger axis: 0 is horizontal,
/// 90 is vertical, measured counter-clockwise on screen (45 runs from lower-left to upper-right).
/// Ignored for other gestures.
/// </summary>
public static (List<IReadOnlyList<PointerPoint>> ContactPaths, List<PointerPoint> Points, int Fingers) PlanTouch(
TouchGesture gesture,
PointerPoint start,
PointerPoint? end,
int distance,
int fingers,
string? direction,
double angleDegrees)
{
var contactPaths = new List<IReadOnlyList<PointerPoint>>();

Expand All @@ -112,14 +128,23 @@ public static (List<IReadOnlyList<PointerPoint>> ContactPaths, List<PointerPoint
case TouchGesture.Stretch:
{
fingers = Math.Max(2, fingers);
int half = Math.Max(PinchCenterGapPx + 1, distance / 2);

// Two opposing fingers along the x-axis. Pinch converges toward the center; stretch
// diverges away from it.
var leftApart = new PointerPoint(start.X - half, start.Y);
var rightApart = new PointerPoint(start.X + half, start.Y);
var leftNear = new PointerPoint(start.X - PinchCenterGapPx, start.Y);
var rightNear = new PointerPoint(start.X + PinchCenterGapPx, start.Y);
// At least 2 px beyond the center gap, so rounding at any angle can't collapse a
// finger's travel to zero.
int half = Math.Max(PinchCenterGapPx + 2, distance / 2);

// Two opposing fingers along the angled axis. Pinch converges toward the center;
// stretch diverges away from it. Screen Y grows downward, so a counter-clockwise
// angle subtracts from Y.
double radians = angleDegrees * Math.PI / 180.0;
double cos = Math.Cos(radians);
double sin = Math.Sin(radians);
var apart = Offset(half, cos, sin);
var near = Offset(PinchCenterGapPx, cos, sin);

var leftApart = new PointerPoint(start.X - apart.X, start.Y - apart.Y);
var rightApart = new PointerPoint(start.X + apart.X, start.Y + apart.Y);
var leftNear = new PointerPoint(start.X - near.X, start.Y - near.Y);
var rightNear = new PointerPoint(start.X + near.X, start.Y + near.Y);

if (gesture == TouchGesture.Pinch)
{
Expand Down Expand Up @@ -157,6 +182,11 @@ public static (List<IReadOnlyList<PointerPoint>> ContactPaths, List<PointerPoint
return (contactPaths, points, contactPaths.Count);
}

private static PointerPoint Offset(int length, double cos, double sin)
=> new(
(int)Math.Round(length * cos, MidpointRounding.AwayFromZero),
(int)Math.Round(-length * sin, MidpointRounding.AwayFromZero));

/// <summary>
/// Computes a swipe end point from <paramref name="start"/> by moving <paramref name="distance"/>
/// pixels along <paramref name="direction"/> (right/left/up/down; defaults to right).
Expand Down
Loading