From 1c651848cc3f3baec8fa0bcfc68faebd3e58c069 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 7 Oct 2013 10:30:20 +0100 Subject: [PATCH] sna/trapezoids: Restore overflow check The range can indeed overflow, so promote the assertion back into a check. Signed-off-by: Chris Wilson --- src/sna/sna_trapezoids_imprecise.c | 7 ++++--- src/sna/sna_trapezoids_precise.c | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/sna/sna_trapezoids_imprecise.c b/src/sna/sna_trapezoids_imprecise.c index 2cfab805..6ca3fedb 100644 --- a/src/sna/sna_trapezoids_imprecise.c +++ b/src/sna/sna_trapezoids_imprecise.c @@ -467,7 +467,8 @@ polygon_init(struct polygon *polygon, unsigned num_buckets = EDGE_Y_BUCKET_INDEX(ymax+EDGE_Y_BUCKET_HEIGHT-1, ymin); - assert(ymax-ymin < 0x7FFFFFFFU - EDGE_Y_BUCKET_HEIGHT); + if (unlikely(ymax - ymin > 0x7FFFFFFFU - EDGE_Y_BUCKET_HEIGHT)) + return false; polygon->edges = polygon->edges_embedded; polygon->y_buckets = polygon->y_buckets_embedded; @@ -936,8 +937,8 @@ tor_init(struct tor *converter, const BoxRec *box, int num_edges) active_list_reset(converter->active); if (!polygon_init(converter->polygon, num_edges, - box->y1 * FAST_SAMPLES_Y, - box->y2 * FAST_SAMPLES_Y)) { + (int)box->y1 * FAST_SAMPLES_Y, + (int)box->y2 * FAST_SAMPLES_Y)) { cell_list_fini(converter->coverages); return false; } diff --git a/src/sna/sna_trapezoids_precise.c b/src/sna/sna_trapezoids_precise.c index 345922f4..bfafca03 100644 --- a/src/sna/sna_trapezoids_precise.c +++ b/src/sna/sna_trapezoids_precise.c @@ -481,7 +481,8 @@ polygon_init(struct polygon *polygon, unsigned num_buckets = EDGE_Y_BUCKET_INDEX(ymax+EDGE_Y_BUCKET_HEIGHT-1, ymin); - assert(ymax - ymin < 0x7FFFFFFFU - EDGE_Y_BUCKET_HEIGHT); + if (unlikely(ymax - ymin > 0x7FFFFFFFU - EDGE_Y_BUCKET_HEIGHT)) + return false; polygon->edges = polygon->edges_embedded; polygon->y_buckets = polygon->y_buckets_embedded; @@ -942,7 +943,7 @@ tor_init(struct tor *converter, const BoxRec *box, int num_edges) active_list_reset(converter->active); if (!polygon_init(converter->polygon, num_edges, - box->y1 * SAMPLES_Y, box->y2 * SAMPLES_Y)) { + (int)box->y1 * SAMPLES_Y, (int)box->y2 * SAMPLES_Y)) { cell_list_fini(converter->coverages); return false; }