Fix flaky test SegmentReplicationWithRemoteStorePressureIT.testAddRep…

…licaWhileWritesBlocked. (#9501)

This test fails on ensureGreen after adding a replica. This is run inside of the try with resources that blocks operations.  The block works by mocking transport calls to prevent segrep from completing until released.

Fixed by moving the ensureGreen until after releasing blockOperations. Also re
duced the doc count that is used while indexing down from max 200.  Writes wit
h the remote store version of this test take a much longer time to execute whe
n performed serially, and we don't need this many docs indexed to create needed checkpoints.

Signed-off-by: Marc Handalian <handalm@amazon.com>

server/src/internalClusterTest/java/org/opensearch/index/SegmentReplicationPressureIT.java

@@ -139,7 +139,6 @@ public void testWritesRejected() throws Exception {
      * This test ensures that a replica can be added while the index is under write block.
      * Ensuring that only write requests are blocked.
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8887")
     public void testAddReplicaWhileWritesBlocked() throws Exception {
         final String primaryNode = internalCluster().startNode();
         createIndex(INDEX_NAME);
@@ -176,10 +175,10 @@ public void testAddReplicaWhileWritesBlocked() throws Exception {
                     .prepareUpdateSettings(INDEX_NAME)
                     .setSettings(Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, 2))
             );
-            ensureGreen(INDEX_NAME);
             replicaNodes.add(replica_2);
-            waitForSearchableDocs(totalDocs.get(), replica_2);
         }
+        ensureGreen(INDEX_NAME);
+        waitForSearchableDocs(totalDocs.get(), replicaNodes);
         refresh(INDEX_NAME);
         // wait for the replicas to catch up after block is released.
         assertReplicaCheckpointUpdated(primaryShard);
@@ -347,7 +346,7 @@ private BulkResponse executeBulkRequest(List<String> nodes, int docsPerBatch) {
     private int indexUntilCheckpointCount() {
         int total = 0;
         for (int i = 0; i < MAX_CHECKPOINTS_BEHIND; i++) {
-            final int numDocs = randomIntBetween(1, 100);
+            final int numDocs = randomIntBetween(1, 5);
             for (int j = 0; j < numDocs; ++j) {
                 indexDoc();
             }