Fix the span information created by TokenStream's FromStr impl

src/libproc_macro/lib.rs

@@ -93,11 +93,10 @@ impl FromStr for TokenStream {
             let src = src.to_string();
             let name = FileName::ProcMacroSourceCode;
             let expn_info = mark.expn_info().unwrap();
-            let call_site = expn_info.call_site;
             // notify the expansion info that it is unhygienic
             let mark = Mark::fresh(mark);
             mark.set_expn_info(expn_info);
-            let span = call_site.with_ctxt(SyntaxContext::empty().apply_mark(mark));
+            let span = DUMMY_SP.with_ctxt(SyntaxContext::empty().apply_mark(mark));
             let stream = parse::parse_stream_from_source_str(name, src, sess, Some(span));
             Ok(__internal::token_stream_wrap(stream))
         })

src/libsyntax/parse/lexer/mod.rs

@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use ast::{self, Ident};
-use syntax_pos::{self, BytePos, CharPos, Pos, Span, NO_EXPANSION};
+use syntax_pos::{self, BytePos, CharPos, Pos, Span, DUMMY_SP};
 use codemap::{CodeMap, FilePathMapping};
 use errors::{FatalError, DiagnosticBuilder};
 use parse::{token, ParseSess};
@@ -66,21 +66,38 @@ pub struct StringReader<'a> {
     token: token::Token,
     span: Span,
     open_braces: Vec<(token::DelimToken, Span)>,
-    pub override_span: Option<Span>,
+    override_span: Span,
 }
 
 impl<'a> StringReader<'a> {
     fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span {
-        unwrap_or!(self.override_span, Span::new(lo, hi, NO_EXPANSION))
+        if !self.override_span.source_equal(&DUMMY_SP) {
+            return self.override_span;
+        }
+        Span::new(lo, hi, self.override_span.ctxt())
     }
     fn mk_ident(&self, string: &str) -> Ident {
         let mut ident = Ident::from_str(string);
-        if let Some(span) = self.override_span {
-            ident.ctxt = span.ctxt();
-        }
+        ident.ctxt = self.override_span.ctxt();
         ident
     }
 
+    /// Set the override for the spans generated by this `StringReader`:
+    ///
+    /// * If this span is a `DUMMY_SP` with no syntax context, the override will
+    ///   have no effect.
+    /// * If the span is a `DUMMY_SP` with an syntax context, the expansion
+    ///   context of the generated spans will be set.
+    /// * If the span is not DUMMY_SP, the span of all tokens will be overridden
+    ///   with that span.
+    pub fn override_span(&mut self, span: Span) {
+        self.override_span = span;
+        // re-generate our peek span with the correct span context and override
+        // information.
+        self.peek_span = self.mk_sp(self.peek_span.lo(), self.peek_span.hi());
+        self.span = self.mk_sp(self.span.lo(), self.span.hi());
+    }
+
     fn next_token(&mut self) -> TokenAndSpan where Self: Sized {
         let res = self.try_next_token();
         self.unwrap_or_abort(res)
@@ -195,7 +212,7 @@ impl<'a> StringReader<'a> {
             token: token::Eof,
             span: syntax_pos::DUMMY_SP,
             open_braces: Vec::new(),
-            override_span: None,
+            override_span: syntax_pos::DUMMY_SP,
         }
     }
 

src/libsyntax/parse/mod.rs

@@ -227,7 +227,9 @@ fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option<Span>)
 pub fn filemap_to_stream(sess: &ParseSess, filemap: Lrc<FileMap>, override_span: Option<Span>)
                          -> TokenStream {
     let mut srdr = lexer::StringReader::new(sess, filemap);
-    srdr.override_span = override_span;
+    if let Some(sp) = override_span {
+        srdr.override_span(sp);
+    }
     srdr.real_token();
     panictry!(srdr.parse_all_token_trees())
 }

src/test/run-pass-fulldeps/proc-macro/span-api-tests.rs

@@ -37,7 +37,7 @@ reemit_legacy! {
 say_hello_extern! { assert_fake_source_file }
 
 reemit! {
-    assert_source_file! { "Hello, world!" }
+    assert_fake_source_file! { "Hello, world!" }
 }
 
 fn main() {}