diff --git a/base32.go b/base32.go
index 99db4a4..1ff1422 100644
--- a/base32.go
+++ b/base32.go
@@ -6,10 +6,8 @@
 package base32
 
 import (
-	"bytes"
 	"io"
 	"strconv"
-	"strings"
 )
 
 /*
@@ -67,13 +65,6 @@ var HexEncoding = NewEncoding(encodeHex)
 var RawStdEncoding = NewEncoding(encodeStd).WithPadding(NoPadding)
 var RawHexEncoding = NewEncoding(encodeHex).WithPadding(NoPadding)
 
-var removeNewlinesMapper = func(r rune) rune {
-	if r == '\r' || r == '\n' {
-		return -1
-	}
-	return r
-}
-
 /*
  * Encoder
  */
@@ -344,18 +335,39 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) {
 // written. If src contains invalid base32 data, it will return the
 // number of bytes successfully written and CorruptInputError.
 // New line characters (\r and \n) are ignored.
-func (enc *Encoding) Decode(dst, src []byte) (n int, err error) {
-	src = bytes.Map(removeNewlinesMapper, src)
-	n, _, err = enc.decode(dst, src)
+func (enc *Encoding) Decode(dst, s []byte) (n int, err error) {
+	// FIXME: if dst is the same as s use decodeInPlace
+	stripped := make([]byte, 0, len(s))
+	for _, c := range s {
+		if c != '\r' && c != '\n' {
+			stripped = append(stripped, c)
+		}
+	}
+	n, _, err = enc.decode(dst, stripped)
+	return
+}
+
+func (enc *Encoding) decodeInPlace(strb []byte) (n int, err error) {
+	off := 0
+	for _, b := range strb {
+		if b == '\n' || b == '\r' {
+			continue
+		}
+		strb[off] = b
+		off++
+	}
+	n, _, err = enc.decode(strb, strb[:off])
 	return
 }
 
 // DecodeString returns the bytes represented by the base32 string s.
 func (enc *Encoding) DecodeString(s string) ([]byte, error) {
-	s = strings.Map(removeNewlinesMapper, s)
-	dbuf := make([]byte, enc.DecodedLen(len(s)))
-	n, _, err := enc.decode(dbuf, []byte(s))
-	return dbuf[:n], err
+	strb := []byte(s)
+	n, err := enc.decodeInPlace(strb)
+	if err != nil {
+		return nil, err
+	}
+	return strb[:n], nil
 }
 
 type decoder struct {